Commit 3fe566b7 authored by IKEDA Soji's avatar IKEDA Soji
Browse files

[bug] Issue #11: Spurious error on duplicate keys with admin sync

  - Now sync_include_admin() passes cache and updates admin_table directly.
  - It no longer reload owner/editor in list config.

Known bug: Big datasource consumes large memory.
parent 91cb25c2
......@@ -7019,309 +7019,171 @@ sub on_the_fly_sync_include {
}
sub sync_include_admin {
my ($self) = shift;
my $option = shift;
my $name = $self->{'name'};
$log->syslog('debug2', '(%s)', $name);
$log->syslog('debug2', '(%s)', @_);
my $self = shift;
## don't care about listmaster role
# don't care about listmaster role.
foreach my $role ('owner', 'editor') {
# Load a hash with the old admin users.
my $old_admin_users = {
map { ($_->{'email'} => $_) }
grep { $_ and $_->{'role'} and $_->{'role'} eq $role }
@{$self->_get_admins || []}
};
## Load a hash with the new admin user list from an include source(s)
my $new_admin_users_include;
## Load a hash with the new admin user users from the list config
my $new_admin_users_config;
unless ($option and $option eq 'purge') {
my $result = $self->_load_list_admin_from_include($role) || {};
$new_admin_users_include = $result->{users};
my @depend_on = @{$result->{depend_on} || []};
## If include sources were not available, do not update admin
## users
## Use DB cache instead
unless (defined $new_admin_users_include) {
$log->syslog('err',
'Could not get %ss from an include source for list %s',
$role, $self);
Sympa::send_notify_to_listmaster($self,
'sync_include_admin_failed', {});
return undef;
}
$new_admin_users_config =
$self->_load_list_admin_from_config($role);
unless (defined $new_admin_users_config) {
$log->syslog('err',
'Could not get %ss from config for list %s',
$role, $name);
return undef;
}
# Update inclusion dependency (added on 6.2.16).
$self->_update_inclusion_table($role, @depend_on);
}
my @add_tab;
my $admin_users_added = 0;
my $admin_users_updated = 0;
return undef
unless $self->_sync_include_user($role);
}
## Get an Exclusive lock
my $lock_fh =
Sympa::LockedFile->new($self->{'dir'} . '/include_admin_user',
20, '+');
unless ($lock_fh) {
$log->syslog('err', 'Could not create new lock');
return undef;
}
$self->_cache_publish_expiry('admin_user');
$self->_cache_publish_expiry('last_sync_admin_user');
## Go through new admin_users_include
foreach my $email (keys %{$new_admin_users_include}) {
return scalar @{$self->get_admins('owner')};
}
# included and subscribed
if (defined $new_admin_users_config->{$email}) {
my $param;
foreach my $p ('reception', 'visibility', 'gecos', 'info',
'profile') {
# config parameters have priority on include parameters
# in case of conflict
$param->{$p} = $new_admin_users_config->{$email}{$p}
if (defined $new_admin_users_config->{$email}{$p});
$param->{$p} ||= $new_admin_users_include->{$email}{$p};
}
sub _sync_include_user {
my $self = shift;
my $role = shift;
#Admin User was already in the DB
if (defined $old_admin_users->{$email}) {
$param->{'included'} = 1;
$param->{'id'} = $new_admin_users_include->{$email}{'id'};
$param->{'subscribed'} = 1;
my $param_update =
is_update_param($param, $old_admin_users->{$email});
# updating
if (defined $param_update) {
if (%{$param_update}) {
$log->syslog('debug', 'Updating %s %s to list %s',
$role, $email, $name);
$param_update->{'update_date'} = time;
unless (
$self->update_list_admin(
$email, $role, $param_update
)
) {
$log->syslog('err',
'(%s) Failed to update %s %s',
$name, $role, $email);
next;
}
$admin_users_updated++;
}
}
# for the next foreach (sort of new_admin_users_config
# that are not included)
delete($new_admin_users_config->{$email});
# Load a hash with the new users.
my $result = $self->_load_list_admin_from_include($role) || {};
my $new_users = $result->{users};
my @depend_on = @{$result->{depend_on} || []};
# add a new included and subscribed admin user
} else {
$log->syslog('debug2', 'Adding %s %s to list %s',
$email, $role, $name);
# If include sources were not available, do not update users.
# Use DB cache instead and warn the listmaster.
unless (defined $new_users) {
$log->syslog('err',
'Could not get %ss from an include source for list %s',
$role, $self);
Sympa::send_notify_to_listmaster($self, 'sync_include_admin_failed',
{});
return undef;
}
foreach my $key (keys %{$param}) {
$new_admin_users_config->{$email}{$key} =
$param->{$key};
}
$new_admin_users_config->{$email}{'included'} = 1;
$new_admin_users_config->{$email}{'subscribed'} = 1;
push(@add_tab, $new_admin_users_config->{$email});
# Update inclusion dependency (added on 6.2.16).
$self->_update_inclusion_table($role, @depend_on);
# for the next foreach (sort of new_admin_users_config
# that are not included)
delete($new_admin_users_config->{$email});
}
# Get an Exclusive lock.
my $lock_fh =
Sympa::LockedFile->new($self->{'dir'} . '/include_admin_user',
20, '+');
unless ($lock_fh) {
$log->syslog('err', 'Could not create new lock');
return undef;
}
# only included
} else {
my $param = $new_admin_users_include->{$email};
#Admin User was already in the DB
if (defined($old_admin_users->{$email})) {
$param->{'included'} = 1;
$param->{'id'} = $new_admin_users_include->{$email}{'id'};
$param->{'subscribed'} = 0;
my $param_update =
is_update_param($param, $old_admin_users->{$email});
# updating
if (defined $param_update) {
if (%{$param_update}) {
$log->syslog('debug', 'Updating %s %s to list %s',
$role, $email, $name);
$param_update->{'update_date'} = time;
unless (
$self->update_list_admin(
$email, $role, $param_update
)
) {
$log->syslog('err',
'(%s) Failed to update %s %s',
$name, $role, $email);
next;
}
$admin_users_updated++;
}
}
# add a new included admin user
} else {
$log->syslog('debug2', 'Adding %s %s to list %s',
$role, $email, $name);
my (%users_added, %users_updated, $users_deleted);
my $time = time;
my $sdm = Sympa::DatabaseManager->instance;
my $sth;
foreach my $key (keys %{$param}) {
$new_admin_users_include->{$email}{$key} =
$param->{$key};
}
$new_admin_users_include->{$email}{'included'} = 1;
push(@add_tab, $new_admin_users_include->{$email});
}
}
# Go through new admin_users_include
foreach my $user (values %$new_users) {
if ($users_added{$user->{email}} or $users_updated{$user->{email}}) {
next;
}
## Go through new admin_users_config (that are not included : only
## subscribed)
foreach my $email (keys %{$new_admin_users_config}) {
my $param = $new_admin_users_config->{$email};
#Admin User was already in the DB
if (defined($old_admin_users->{$email})) {
$param->{'included'} = 0;
$param->{'id'} = '';
$param->{'subscribed'} = 1;
my $param_update =
is_update_param($param, $old_admin_users->{$email});
# updating
if (defined $param_update) {
if (%{$param_update}) {
$log->syslog('debug', 'Updating %s %s to list %s',
$role, $email, $name);
$param_update->{'update_date'} = time;
unless (
$self->update_list_admin(
$email, $role, $param_update
)
) {
$log->syslog('err', '(%s) Failed to update %s %s',
$name, $role, $email);
next;
}
$admin_users_updated++;
}
}
# add a new subscribed admin user
unless (
$sdm
and $sth = $sdm->do_prepared_query(
q{UPDATE admin_table
SET included_admin = 1, include_sources_admin = ?,
update_epoch_admin = ?
WHERE role_admin = ? AND user_admin = ? AND
list_admin = ? AND robot_admin = ?},
$user->{id}, $time,
$role, $user->{email},
$self->{'name'}, $self->{'domain'}
)
and $sth->rows
) {
unless (
$sdm
and $sth = $sdm->do_prepared_query(
sprintf(
q{INSERT INTO admin_table
(user_admin, comment_admin,
list_admin, robot_admin,
date_admin, update_epoch_admin,
reception_admin, visibility_admin,
subscribed_admin, included_admin,
include_sources_admin,
role_admin, info_admin, profile_admin)
VALUES (?, ?, ?, ?, %s, ?, ?, ?, 0, 1, ?, ?, ?, ?)},
$sdm->get_canonical_write_date($time)
),
$user->{email}, $user->{gecos},
$self->{'name'}, $self->{'domain'},
$time,
$user->{reception}, $user->{visibility},
$user->{id},
$role, $user->{info}, $user->{profile}
)
and $sth->rows
) {
$log->syslog('err', '(%s) Failed to update %s %s',
$self, $role, $user->{email});
} else {
$log->syslog('debug2', 'Adding %s %s to list %s',
$role, $email, $name);
foreach my $key (keys %{$param}) {
$new_admin_users_config->{$email}{$key} = $param->{$key};
}
$new_admin_users_config->{$email}{'subscribed'} = 1;
push(@add_tab, $new_admin_users_config->{$email});
}
}
if ($#add_tab >= 0) {
unless ($admin_users_added =
$self->add_list_admin($role, @add_tab)) {
$log->syslog('err', 'Failed to add new %s(s) to list %s',
$role, $self);
return undef;
$users_added{$user->{email}} = 1;
}
} else {
$users_updated{$user->{email}} = 1;
}
}
if ($admin_users_added) {
$log->syslog('debug', '(%s) %d %s(s) added',
$name, $admin_users_added, $role);
}
$log->syslog('debug', '(%s) %d %s(s) updated',
$name, $admin_users_updated, $role);
## Go though old list of admin users
my $admin_users_removed = 0;
my @deltab;
foreach my $email (keys %$old_admin_users) {
unless (defined($new_admin_users_include->{$email})
|| defined($new_admin_users_config->{$email})) {
$log->syslog('debug2', 'Removing %s %s to list %s',
$role, $email, $name);
push(@deltab, $email);
}
}
$log->syslog(
'debug', '(%s) %d %s(s) added, %d %s(s) updated',
$self, scalar keys %users_added,
$role, scalar keys %users_updated, $role
);
if ($#deltab >= 0) {
unless ($admin_users_removed =
$self->delete_list_admin($role, @deltab)) {
$log->syslog('err', '(%s) Failed to delete %s %s',
$name, $role, $admin_users_removed);
return undef;
}
$log->syslog('debug', '(%s) %d %s(s) removed',
$name, $admin_users_removed, $role);
}
# Go though old list of admin users.
$users_deleted = 0;
unless (
$sdm
and $sth = $sdm->do_prepared_query(
q{DELETE FROM admin_table
WHERE role_admin = ? AND list_admin = ? AND robot_admin = ? AND
(subscribed_admin IS NULL OR subscribed_admin = 0) AND
NOT (included_admin IS NULL OR included_admin = 0) AND
(update_epoch_admin IS NULL OR update_epoch_admin < ?)},
$role, $self->{'name'}, $self->{'domain'},
$time
)
) {
$log->syslog('err', '(%s) Failed to delete %s', $self, $role);
} else {
$users_deleted += $sth->rows;
}
unless (
$sdm
and $sth = $sdm->do_prepared_query(
q{UPDATE admin_table
SET included_admin = 0, include_sources_admin = NULL,
update_epoch_admin = ?
WHERE role_admin = ? AND list_admin = ? AND robot_admin = ? AND
subscribed_admin = 1 AND
(update_epoch_admin IS NULL OR update_epoch_admin < ?)},
$time,
$role, $self->{'name'}, $self->{'domain'},
$time
)
) {
$log->syslog('err', '(%s) Failed to delete %s', $self, $role);
} else {
$users_deleted += $sth->rows;
}
## Release lock
unless ($lock_fh->close()) {
return undef;
}
if ($users_deleted) {
$log->syslog('debug', '(%s) %d %s(s) removed',
$self, $users_deleted, $role);
}
$self->_cache_publish_expiry('admin_user');
$self->_cache_publish_expiry('last_sync_admin_user');
# Release lock.
unless ($lock_fh->close()) {
return undef;
}
return scalar @{$self->get_admins('owner')};
return 1;
}
## Load param admin users from the config of the list
sub _load_list_admin_from_config {
my $self = shift;
my $role = shift;
my $name = $self->{'name'};
my %admin_users;
$log->syslog('debug2', '(%s) For list %s', $role, $name);
foreach my $entry (@{$self->{'admin'}{$role}}) {
my $email = lc($entry->{'email'});
my %u;
$u{'email'} = $email;
$u{'reception'} = $entry->{'reception'};
$u{'visibility'} = $entry->{'visibility'};
$u{'gecos'} = $entry->{'gecos'};
$u{'info'} = $entry->{'info'};
$u{'profile'} = $entry->{'profile'} if ($role eq 'owner');
$admin_users{$email} = \%u;
}
return \%admin_users;
}
# No longer used.
#sub _load_list_admin_from_config;
## return true if new_param has changed from old_param
# $new_param is changed to return only entries that need to
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment