Skip to content

GitLab

Unverified Commit 7433eb4c authored by Stefan Hornburg (Racke)'s avatar Stefan Hornburg (Racke) Committed by GitHub
Browse files

Merge pull request #914 from ikedas/issue-907_trial-2 

`sympa.pl --sync_include`: No errors were reported to caller
parents fff73b73 9b107754
Hide whitespace changes
Inline Side-by-side
default/mail_tt2/report.tt2
View file @ 7433eb4c
......@@ -357,7 +357,43 @@
[%~ ELSIF report_entry == 'no_message_to_moderate' ~%]
[%|loc(report_param.listname)%]There is no message to moderate for list '%1'.[%END%]
[%~ ELSIF report_entry == 'time_limit_for_include' ~%]
[%~ ELSIF report_entry == 'include_skip' ~%]
[% IF report_param.role == 'owner' ~%]
[%|loc(report_param.listname)%]Including list owners into list %2 was skipped.[%END%]
[%~ ELSIF report_param.role == 'editor' ~%]
[%|loc(report_param.listname)%]Including moderators into list %2 was skipped.[%END%]
[%~ ELSE ~%]
[%|loc(report_param.listname)%]Including subscribers into list %2 was skipped.[%END%]
[%~ END %]
[%~ ELSIF report_entry == 'include' ~%]
[% IF report_param.role == 'owner' ~%]
[%|loc(report_param.name,report_param.listname)%]List owners were included from data source "%1" into list %2.[%END%]
[%~ ELSIF report_param.role == 'editor' ~%]
[%|loc(report_param.name,report_param.listname)%]Moderators were included from data source "%1" into list %2.[%END%]
[%~ ELSE ~%]
[%|loc(report_param.name,report_param.listname)%]Subscribers were included from data source "%1" into list %2.[%END%]
[%~ END %]
[%~ ELSIF report_entry == 'include_failed' ~%]
[% IF report_param.role == 'owner' ~%]
[%|loc(report_param.name,report_param.listname)%]Including list owners from data source "%1" into list %2 failed.[%END%]
[%~ ELSIF report_param.role == 'editor' ~%]
[%|loc(report_param.name,report_param.listname)%]Including moderators from data source "%1" into list %2 failed.[%END%]
[%~ ELSE ~%]
[%|loc(report_param.name,report_param.listname)%]Including subscribers from data source "%1" into list %2 failed.[%END%]
[%~ END %]
[%~ ELSIF report_entry == 'include_incomplete' ~%]
[%|loc(report_param.result.added,report_param.result.held,report_param.result.updated)%]Inclusion entirely failed (%1 added, %2 held, %3 updated).[%END%]
[%~ ELSIF report_entry == 'include_partial' ~%]
[%|loc(report_param.result.added,report_param.result.held,report_param.result.updated)%]Inclusion partially failed (%1 added, %2 held, %3 updated).[%END%]
[%~ ELSIF report_entry == 'include_performed' ~%]
[%|loc(report_param.result.added,report_param.result.deleted,report_param.result.updated)%]Inclusion succeeded (%1 added, %2 deleted, %3 updated).[%END%]
[%~ ELSIF report_entry == 'time_limit_for_include' # Maybe no longer used ~%]
[%|loc()%]You're not allowed to make sync, time limit in action[%END%]
[%~ ELSIF report_entry == 'del_performed' ~%]
......
src/lib/Sympa/Request/Handler/include.pm
View file @ 7433eb4c
......@@ -141,13 +141,15 @@ sub _twist {
unless ($lock_fh) {
$log->syslog('info', '%s: Locked, skip inclusion', $list);
$self->add_stash($request, 'notice', 'include_skip',
{listname => $list->{'name'}});
{listname => $list->{'name'}, role => $role});
return 0;
}
# I. Start.
my (%start_times, $last_start_time, $start_time);
my (%start_times, $start_time);
my $last_start_time;
seek $lock_fh, 0, 0;
while (my $line = <$lock_fh>) {
next unless $line =~ /\A(\w+)\s+(\d+)/;
......@@ -162,7 +164,7 @@ sub _twist {
# Avoid retrace of clock e.g. by outage of NTP server.
$log->syslog('info', '%s: Clock got behind, skip inclusion', $list);
$self->add_stash($request, 'notice', 'include_skip',
{listname => $list->{'name'}});
{listname => $list->{'name'}, role => $role});
return 0;
}
......@@ -176,16 +178,30 @@ sub _twist {
# II. Include new entries.
my %result = (added => 0, deleted => 0, updated => 0, kept => 0);
my %result =
(added => 0, deleted => 0, updated => 0, kept => 0, held => 0);
my $succeeded = 0;
foreach my $ds (@{$dss || []}) {
$lock_fh->extend;
next unless $ds->is_allowed_to_sync;
my %res = _update_users($ds, $start_time);
next unless %res;
unless (%res) {
$self->add_stash(
$request, 'notice',
'include_failed',
{ listname => $list->{'name'},
role => $role,
id => $ds->get_short_id,
name => $ds->name,
}
);
next;
}
# Update time of allowed and succeeded data sources.
$start_times{$ds->get_short_id} = $start_time;
$succeeded++;
# Special treatment for Sympa::DataSource::List.
_update_inclusion_table($ds, $start_time)
......@@ -199,6 +215,7 @@ sub _twist {
$request, 'notice',
'include',
{ listname => $list->{'name'},
role => $role,
id => $ds->get_short_id,
name => $ds->name,
result => {%res}
......@@ -211,21 +228,18 @@ sub _twist {
# III. Expire outdated entries.
# Choose most earlier time of succeeding inclusions (if any of
# data sources have not succeeded yet, time is not defined).
$last_start_time = $start_time;
foreach my $id (map { $_->get_short_id } @$dss) {
unless (defined $start_times{$id}) {
undef $last_start_time;
last;
} elsif ($start_times{$id} < $last_start_time) {
$last_start_time = $start_times{$id};
}
}
if (defined $last_start_time) {
if ($succeeded == scalar @$dss) {
# All data sources succeeded.
$lock_fh->extend;
# Choose most earlier time of succeeding inclusions (if any of
# data sources have not succeeded yet, time is not known).
my $last_start_time = $start_time;
foreach my $id (map { $_->get_short_id } @$dss) {
$last_start_time = $start_times{$id}
if $start_times{$id} < $last_start_time;
}
my %res = _expire_users($list, $role, $last_start_time);
unless (%res) {
$self->add_stash($request, 'intern');
......@@ -238,6 +252,21 @@ sub _twist {
# Special treatment for Sympa::DataSource::List.
_expire_inclusion_table($list, $role, $last_start_time);
} else {
# Part(s) or entire data sources failed.
$lock_fh->extend;
# Estimate number of held users, i.e. users not decided to
# delete, update nor keep.
my %res = _expire_users($list, $role, $start_time, dry_run => 1);
unless (%res) {
$self->add_stash($request, 'intern');
#FIMXE: Report error.
return undef;
}
foreach my $key (keys %res) {
$result{$key} += $res{$key} if exists $result{$key};
}
}
# IV. Update custom attributes.
......@@ -275,12 +304,34 @@ sub _twist {
}
unlink $lock_file . '.old';
$log->syslog(
'info', '%s: %d included, %d deleted, %d updated',
$request, @result{qw(added deleted updated)}
);
$self->add_stash($request, 'notice', 'include_performed',
{listname => $list->{'name'}, result => {%result}});
if ($succeeded == scalar @$dss) {
# All data sources succeeded.
$log->syslog(
'info', '%s: Success, %d added, %d deleted, %d updated',
$request, @result{qw(added deleted updated)}
);
$self->add_stash($request, 'notice', 'include_performed',
{listname => $list->{'name'}, role => $role, result => {%result}}
);
} elsif ($succeeded) {
# Part(s) of data sources failed.
$log->syslog(
'info', '%s: Partial, %d added, %d held, %d updated',
$request, @result{qw(added held updated)}
);
$self->add_stash($request, 'notice', 'include_partial',
{listname => $list->{'name'}, role => $role, result => {%result}}
);
} else {
# All data sources failed.
$log->syslog(
'info', '%s: Failure, %d added, %d held, %d updated',
$request, @result{qw(added held updated)}
);
$self->add_stash($request, 'notice', 'include_incomplete',
{listname => $list->{'name'}, role => $role, result => {%result}}
);
}
return 1;
}
......@@ -453,6 +504,7 @@ sub _expire_users {
my $list = shift;
my $role = shift;
my $last_start_time = shift;
my %options = @_;
my $sdm = Sympa::DatabaseManager->instance;
return unless $sdm;
......@@ -462,6 +514,26 @@ sub _expire_users {
? ('subscriber', '')
: ('admin', sprintf ' AND role_admin = %s', $sdm->quote($role));
if ($options{dry_run}) {
unless (
$sth = $sdm->do_prepared_query(
qq{SELECT COUNT(*)
FROM ${t}_table
WHERE (subscribed_$t IS NULL OR subscribed_$t <> 1) AND
inclusion_$t IS NOT NULL AND inclusion_$t < ? AND
list_$t = ? AND robot_$t = ?$r},
$last_start_time,
$list->{'name'}, $list->{'domain'}
)
) {
return undef;
}
my ($count) = $sth->fetchrow_array;
$sth->finish;
return (held => ($count || 0));
}
my $deleted = 0;
# Remove list users not subscribing (only included) and
# not included anymore.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment