Unverified Commit 91bfffa6 authored by IKEDA Soji's avatar IKEDA Soji Committed by GitHub
Browse files

Merge pull request #267 from ikedas/issue-232 by ikedas

sympa.pl: Improving --dump_users and adding --restore_users, and related fixes
parents eee733bf 91cb25c2
......@@ -10,9 +10,9 @@
[% END %]
<a class="actionMenuLinks" href="[% 'reviewbouncing' | url_rel([list]) %]">[%|loc%]Bounces[%END%]</a>
[% IF action == 'search' %]
<a class="actionMenuLinks" href="[% 'dump' | url_rel([list,filter]) %]">[%|loc%]Dump[%END%]</a>
<a class="actionMenuLinks" href="[% 'export_member' | url_rel([list],{filter=>filter}) %]">[%|loc%]Dump[%END%]</a>
[% ELSE %]
<a class="actionMenuLinks" href="[% 'dump' | url_rel([list,'light']) %]">[%|loc%]Dump[%END%]</a>
<a class="actionMenuLinks" href="[% 'export_member' | url_rel([list,'light']) %]">[%|loc%]Dump[%END%]</a>
[% END %]
<a class="actionMenuLinks" href="[% 'show_exclude' | url_rel([list]) %]">[%|loc%]Exclude[%END%]</a>
<br />
......
......@@ -2,7 +2,7 @@
<h2>[%|loc%]Manage bouncing list members[%END%] <a href="[% 'nomenu/help/admin' | url_rel %]#manage_bounces" title="[%|loc%]Open in a new window[%END%]" onclick="window.open('','wws_help','toolbar=no,location=no,directories=no,status=no,menubar=no,scrollbars=yes,resizable=yes,copyhistory=no,width=400,height=200')" target="wws_help"><i class="fa fa-question-circle" title="[%|loc%]Help[%END%]"></i></a></h2>
<br />
[% IF bounce_rate %]
<a class="actionMenuLinks" href="[% 'dump' | url_rel([list,'bounce']) %]">[%|loc%]Dump[%END%]</a>
<a class="actionMenuLinks" href="[% 'export_member' | url_rel([list,'bounce']) %]">[%|loc%]Dump[%END%]</a>
<form action="[% path_cgi %]" method="post">
<fieldset>
......
......@@ -278,7 +278,7 @@ our %comm = (
'd_set_owner' => 'do_d_set_owner',
'd_admin' => 'do_d_admin',
'dump_scenario' => 'do_dump_scenario',
'dump' => 'do_dump',
'export_member' => 'do_export_member',
'remind' => 'do_remind',
'move_user' => 'do_move_user',
'load_cert' => 'do_load_cert',
......@@ -337,6 +337,7 @@ my %comm_aliases = (
'automatic_lists_request' => 'create_automatic_list_request',
'change_email' => 'move_user',
'change_email_request' => 'move_user',
'dump' => 'export_member',
'ignoresig' => 'decl_del',
'ignoresub' => 'decl_add',
'del_fromsig' => 'auth_del',
......@@ -452,7 +453,7 @@ our %action_args = (
'd_control' => ['list', '@path'],
'd_change_access' => ['list', '@path'],
'd_set_owner' => ['list', '@path'],
'dump' => ['list', 'format'],
'export_member' => ['list', 'format'],
'search' => ['list', 'filter'],
'search_user' => ['email'],
'set_lang' => ['lang'],
......@@ -554,13 +555,13 @@ our %required_args = (
'delete_pictures' => ['param.list', 'param.user.email'],
'distribute' => ['param.list', 'param.user.email', 'id|idspam'],
'add_frommod' => ['param.list', 'param.user.email', 'id'],
'dump' => ['param.list'],
'dump_scenario' => ['param.list', 'pname'],
'edit_list' => ['param.user.email', 'param.list'],
'edit_list_request' => ['param.user.email', 'param.list'],
'edit_template' => ['webormail'],
'editfile' => ['param.user.email'],
'editsubscriber' => ['param.list', 'param.user.email', 'email'],
'export_member' => ['param.list'],
'get_closed_lists' => ['param.user.email'],
'get_inactive_lists' => ['param.user.email'],
'get_latest_lists' => ['param.user.email'],
......@@ -748,8 +749,8 @@ my %action_type = (
'd_admin' => 'admin',
'd_reject_shared' => 'admin',
'd_install_shared' => 'admin',
'export_member' => 'admin',
'dump_scenario' => 'admin',
'dump' => 'admin',
'open_list' => 'admin',
'remind' => 'admin',
#'subindex' => 'admin',
......@@ -15908,89 +15909,105 @@ sub do_dump_scenario {
return 1;
}
 
## Subscribers' list
sub do_dump {
wwslog('info', '(%s)', $param->{'list'});
## Whatever the action return, it must never send a complex html page
$param->{'bypass'} = 1;
$param->{'content_type'} = "text/plain";
$param->{'file'} = undef;
# Subscribers' list
# Old name: do_dump().
sub do_export_member {
wwslog('info', '(%s, %s, %s)', $param->{'list'}, $in{'format'},
$in{'filter'});
 
## Access control
unless (defined check_authz('do_dump', 'review')) {
undef $param->{'bypass'};
# Access control
unless (defined check_authz('do_export_member', 'review')) {
return undef;
}
 
$list->dump();
$param->{'file'} = $list->{'dir'} . '/subscribers.db.dump';
if ($in{'format'} eq 'light') {
unless (open(DUMP, $param->{'file'})) {
Sympa::WWW::Report::reject_report_web('intern', 'cannot_open_file',
{'file' => $param->{'file'}},
$param->{'action'}, '', $param->{'user'}{'email'}, $robot);
wwslog('info', 'Unable to open file %s', $param->{'file'});
return undef;
}
unless (open(LIGHTDUMP, ">$param->{'file'}.light")) {
Sympa::WWW::Report::reject_report_web(
'intern',
'cannot_open_file',
{'file' => "$param->{'file'}.light"},
$param->{'action'},
'',
$param->{'user'}{'email'},
$robot
);
wwslog('err', 'Unable to create file %s.light', $param->{'file'});
return undef;
}
while (<DUMP>) {
next unless ($_ =~ /^email\s(.*)/);
print LIGHTDUMP "$1\n";
}
close LIGHTDUMP;
close DUMP;
$param->{'file'} = "$list->{'dir'}/subscribers.db.dump.light";
my $format = $in{'format'} || 'full';
my $filter = $in{'filter'};
$filter = '' unless defined $filter;
 
$param->{'bypass'} = 'extreme';
printf "Content-Type: text/plain; Charset=\"UTF-8\"; name=\"%s.txt\"\n"
. "Content-Disposition: attachment; filename=\"%s.txt\"\n"
. "Content-Transfer-Encoding: 8BIT\n" . "\n",
$list->get_id, $list->get_id;
if ($format eq 'bounce') {
print '# '
. join("\t",
'Email',
'Name',
'Bounce score',
'Bounce count',
'First bounce',
'Last bounce')
. "\n";
} elsif ($format eq 'light') {
;
} else {
$param->{'file'} = "$list->{'dir'}/select.dump";
wwslog('info', 'Opening %s', $param->{'file'});
printf "# Exported subscribers with search filter \"%s\"\n", $filter;
}
my $searchkey = Sympa::Tools::Text::foldcase($filter)
if defined $filter and length $filter;
 
unless (open(DUMP, ">$param->{'file'}")) {
Sympa::WWW::Report::reject_report_web('intern', 'file_update_failed',
{}, $param->{'action'}, '', $param->{'user'}{'email'},
$robot);
wwslog('err', 'Unable to create file %s', $param->{'file'});
return undef;
}
for (
my $subscriber = _subscriber_first($list, type => $format);
$subscriber;
$subscriber = _subscriber_next($list, type => $format)
) {
my $email = $subscriber->{email};
my $gecos = $subscriber->{gecos};
next unless defined $email and length $email; # malformed record.
 
if ($in{'format'} eq 'bounce') {
$in{'size'} = 'all';
do_reviewbouncing();
print DUMP "# Exported bouncing subscribers\n";
print DUMP
"# Email\t\tName\tBounce score\tBounce count\tFirst bounce\tLast bounce\n";
foreach my $user (@{$param->{'members'}}) {
print DUMP
"$user->{'email'}\t$user->{'gecos'}\t$user->{'bounce_score'}\t$user->{'bounce_count'}\t$user->{'first_bounce'}\t$user->{'last_bounce'}\n";
}
if (defined $searchkey and length $searchkey) {
my $e = Sympa::Tools::Text::foldcase($email);
my $g = Sympa::Tools::Text::foldcase($gecos);
next
unless 0 <= index $e, $searchkey
or 0 <= index $g, $searchkey;
}
if ($format eq 'bounce') {
print join "\t",
$email, $gecos,
@{$subscriber}
{qw(bounce_score bounce_count first_bounce last_bounce)};
print "\n";
} elsif ($format eq 'light') {
print "$email\n";
} else {
$in{'filter'} = $in{'format'};
do_search();
print DUMP
"# Exported subscribers with search filter \"$in{'format'}\"\n";
foreach my $user (@{$param->{'members'}}) {
print DUMP "$user->{'email'}\t$user->{'gecos'}\n";
}
print join "\t", $email, $gecos;
print "\n";
}
close DUMP;
}
return 1;
}
 
sub _subscriber_first {
my $list = shift;
my %options = @_;
if ($options{type} and $options{type} eq 'bounce') {
my $i = $list->get_first_bouncing_list_member;
$list->parse_list_member_bounce($i) if $i;
return $i;
} else {
return $list->get_first_list_member;
}
}
sub _subscriber_next {
my $list = shift;
my %options = @_;
if ($options{type} and $options{type} eq 'bounce') {
my $i = $list->get_next_bouncing_list_member;
$list->parse_list_member_bounce($i) if $i;
return $i;
} else {
return $list->get_next_list_member;
}
}
## returns a mailto according to list spam protection parameter
# No longer used.
#sub mailto;
......
......@@ -142,6 +142,12 @@ Delete the indicated users from the list.
=item delete_list_admin ( ROLE, ARRAY )
Delete the indicated admin user with the predefined role from the list.
ROLE may be C<'owner'> or C<'editor'>.
=item dump_users ( ROLE )
Dump user information in user store into file C<I<$role>.dump> under
list directory. ROLE may be C<'member'>, C<'owner'> or C<'editor'>.
=item get_cookie ()
......@@ -198,6 +204,11 @@ the list.
OBSOLETED.
Use get_admins().
=item restore_users ( ROLE )
Import user information into user store from file C<I<$role>.dump> under
list directory. ROLE may be C<'member'>, C<'owner'> or C<'editor'>.
=item update_list_member ( $email, key =E<gt> value, ... )
I<Instance method>.
......@@ -695,24 +706,60 @@ sub _cache_put {
# Moved to: Sympa::Spindle::DistributeMessage::_extract_verp_rcpt().
#sub _extract_verp_rcpt;
## Dumps a copy of lists to disk, in text format
sub dump {
# Dumps a copy of list users to disk, in text format.
# Old name: Sympa::List::dump() which dumped only members.
sub dump_users {
$log->syslog('debug2', '(%s, %s)', @_);
my $self = shift;
$log->syslog('debug2', '(%s)', $self->{'name'});
my $role = shift;
die 'bug in logic. Ask developer'
unless grep {$role eq $_} qw(member owner editor);
unless (defined $self) {
$log->syslog('err', 'Unknown list');
my $file = $self->{'dir'} . '/' . $role . '.dump';
unlink $file . '.old' if -e $file . '.old';
rename $file, $file . '.old' if -e $file;
my $lock_fh = Sympa::LockedFile->new($file, 5, '>');
unless ($lock_fh) {
$log->syslog('err', 'Failed to save file %s.new: %s', $file,
Sympa::LockedFile->last_error);
return undef;
}
my $user_file_name = "$self->{'dir'}/subscribers.db.dump";
if ($role eq 'member') {
my %map_field = _map_list_member_cols();
unless ($self->_save_list_members_file($user_file_name)) {
$log->syslog('err', 'Failed to save file %s', $user_file_name);
return undef;
my $user;
for (
$user = $self->get_first_list_member();
$user;
$user = $self->get_next_list_member()
) {
foreach my $k (sort keys %map_field) {
printf $lock_fh "%s %s\n", $k, $user->{$k}
if defined $user->{$k} and length $user->{$k};
}
print $lock_fh "\n";
}
} else {
foreach my $user (@{$self->_get_admins || []}) {
next unless $user->{role} eq $role;
foreach my $k (
qw(date update_date email gecos profile
reception visibility info
subscribed included id)
) {
printf $lock_fh "%s %s\n", $k, $user->{$k}
if defined $user->{$k} and length $user->{$k};
}
print $lock_fh "\n";
}
}
# Note: "subscribers" file was deprecated. No need to load "stats" file.
$lock_fh->close;
# FIXME:Are these lines required?
$self->{'_mtime'}{'config'} =
Sympa::Tools::File::get_mtime($self->{'dir'} . '/config');
......@@ -4324,37 +4371,70 @@ sub load_data_sources_list {
# No longer used.
#sub _load_stats_file;
## Loads the list of subscribers.
sub _load_list_members_file {
my $file = shift;
$log->syslog('debug2', '(%s)', $file);
## Loads the list of users.
# Old name:: Sympa::List::_load_list_members_file($file) which loaded members.
sub restore_users {
$log->syslog('debug2', '(%s, %s)', @_);
my $self = shift;
my $role = shift;
## Open the file and switch to paragraph mode.
open(L, $file) || return undef;
die 'bug in logic. Ask developer'
unless grep {$role eq $_} qw(member owner editor);
## Process the lines
local $RS;
my $data = <L>;
my $file = $self->{'dir'} . '/' . $role . '.dump';
my @users;
foreach (split /\n\n/, $data) {
my (%user, $email);
$user{'email'} = $email = $1 if (/^\s*email\s+(.+)\s*$/om);
$user{'gecos'} = $1 if (/^\s*gecos\s+(.+)\s*$/om);
$user{'date'} = $1 if (/^\s*date\s+(\d+)\s*$/om);
$user{'update_date'} = $1 if (/^\s*update_date\s+(\d+)\s*$/om);
$user{'reception'} = $1
if (
/^\s*reception\s+(digest|nomail|summary|notice|txt|html|urlize|not_me)\s*$/om
);
$user{'visibility'} = $1
if (/^\s*visibility\s+(conceal|noconceal)\s*$/om);
# Open the file and switch to paragraph mode.
my $lock_fh = Sympa::LockedFile->new($file, 5, '<') or return;
local $RS = '';
if ($role eq 'member') {
my %map_field = _map_list_member_cols();
push @users, \%user;
while (my $para = <$lock_fh>) {
my $user = {
map {
#FIMXE: Define appropriate schema.
if (/^\s*(suspend|subscribed|included)\s+(\S+)\s*$/) {
($1 => !!$2);
} elsif (/^\s*(date|update_date|startdate|enddate|bounce_score|number_messages)\s+(\d+)\s*$/
or /^\s*(reception)\s+(mail|digest|nomail|summary|notice|txt|html|urlize|not_me)\s*$/
or /^\s*(visibility)\s+(conceal|noconceal)\s*$/
or (/^\s*(\w+)\s+(.+)\s*$/ and $map_field{$1})) {
($1 => $2);
} else {
();
}
} split /\n/, $para
};
next unless $user->{email};
$self->add_list_member($user);
}
} else {
while (my $para = <$lock_fh>) {
my $user = {
map {
#FIMXE:Define appropriate schema.
if (/^\s*(subscribed|included)\s+(\S+)\s*$/) {
($1 => !!$2);
} elsif (/^\s*(email|gecos|info|id)\s+(.+)\s*$/
or /^\s*(profile)\s+(normal|privileged)\s*$/
or /^\s*(date|update_date)\s+(\d+)\s*$/
or /^\s*(reception)\s+(mail|nomail)\s*$/
or /^\s*(visibility)\s+(conceal|noconceal)\s*$/) {
($1 => $2);
} else {
();
}
} split /\n/, $para
};
next unless $user->{email};
$self->add_list_admin($role, $user);
}
}
close(L);
return @users;
$lock_fh->close;
}
## include a remote sympa list as subscribers.
......@@ -7341,35 +7421,8 @@ sub _inclusion_loop {
#sub _load_total_db;
## Writes the user list to disk
sub _save_list_members_file {
my ($self, $file) = @_;
$log->syslog('debug3', '(%s)', $file);
my ($k, $s);
$log->syslog('debug2', 'Saving user file %s', $file);
rename("$file", "$file.old");
open my $fh, '>', $file or return undef;
for (
$s = $self->get_first_list_member();
$s;
$s = $self->get_next_list_member()
) {
foreach $k (
'date', 'update_date', 'email', 'gecos',
'reception', 'visibility'
) {
printf $fh "%s %s\n", $k, $s->{$k}
if defined $s->{$k} and length $s->{$k};
}
print $fh "\n";
}
close $fh;
return 1;
}
# Depreceted. Use Sympa::List::dump_users().
#sub _save_list_members_file;
## Does the real job : stores the message given as an argument into
## the digest of the list.
......
......@@ -154,9 +154,10 @@ sub _close {
Conf::get_robot_conf($list->{'domain'}, 'alias_manager'));
$aliases->del($list) if $aliases;
# Dump subscribers.
$list->_save_list_members_file(
$list->{'dir'} . '/subscribers.closed.dump');
# Dump users.
$list->dump_users('member');
$list->dump_users('owner');
$list->dump_users('editor');
## Delete users
my @users;
......
......@@ -221,8 +221,9 @@ sub _move {
my $aliases = Sympa::Aliases->new(
Conf::get_robot_conf($current_list->{'domain'}, 'alias_manager'));
$aliases->del($current_list) if $aliases;
$current_list->_save_list_members_file(
$current_list->{'dir'} . '/subscribers.closed.dump');
$current_list->dump_users('member');
$current_list->dump_users('owner');
$current_list->dump_users('editor');
# Set list status to pending if creation list is moderated.
# Save config file for the new() later to reload it.
......
......@@ -25,6 +25,7 @@ package Sympa::Request::Handler::open_list;
use strict;
use warnings;
use English qw(-no_match_vars);
use File::Path qw();
use Sympa;
......@@ -80,16 +81,32 @@ sub _twist {
return undef;
}
unless (-f $list->{'dir'} . '/subscribers.closed.dump') {
$log->syslog('notice', 'No subscribers to restore');
# Dump initial permanent owners/editors in config file.
if ($mode eq 'install') {
my ($fh, $fh_config);
foreach my $role (qw(owner editor)) {
my $file = $list->{'dir'} . '/' . $role . '.dump';
my $config = $list->{'dir'} . '/config';
if ( !-e $file
and open($fh, '>', $file)
and open($fh_config, '<', $config)) {
local $RS = ''; # read paragraph by each
my $admins = join '', grep {/\A\s*$role\b/} <$fh_config>;
print $fh $admins;
close $fh;
close $fh_config;
}
}
}
# Load permanent users.
$list->restore_users('member');
$list->restore_users('owner');
$list->restore_users('editor');
# Load initial transitional owners/editors from external data sources.
if ($mode eq 'install') {
$list->sync_include_admin;
}
my @users = Sympa::List::_load_list_members_file(
$list->{'dir'} . '/subscribers.closed.dump');
# Insert users in database.
$list->add_list_member(@users);
# Restore admins
$list->sync_include_admin;
# Install new aliases.
my $aliases = Sympa::Aliases->new(
......
......@@ -746,30 +746,29 @@ sub upgrade {
$list->{'name'}
);
my @users = Sympa::List::_load_list_members_file(
"$list->{'dir'}/subscribers");
$list->{'admin'}{'user_data_source'} = 'include2';
# Load <list dir>/subscribers to the DB
if (-e $list->{'dir'} . '/subscribers'
and rename $list->{'dir'} . '/subscribers',
$list->{'dir'} . '/member.dump'
) {
$list->restore_users('member');
## Add users to the DB
$list->add_list_member(@users);
my $total = $list->{'add_outcome'}{'added_members'};
if (defined $list->{'add_outcome'}{'errors'}) {
$log->syslog(
'err',
'Failed to add users: %s',
$list->{'add_outcome'}{'errors'}{'error_message'}
);
my $total = $list->{'add_outcome'}{'added_members'};
if (defined $list->{'add_outcome'}{'errors'}) {
$log->syslog('err', 'Failed to add users: %s',
$list->{'add_outcome'}{'errors'}
{'error_message'});
}
$log->syslog('notice',
'%d subscribers have been loaded into the database',
$total);
}
$log->syslog('notice',
'%d subscribers have been loaded into the database',
$total);
$list->{'admin'}{'user_data_source'} = 'include2';
unless ($list->save_config('automatic')) {
$log->syslog('err',
'Failed to save config file for list %s',
$list->{'name'});
'Failed to save config file for list %s', $list);
}
} elsif ($list->{'admin'}{'user_data_source'} eq 'database') {
......@@ -943,7 +942,7 @@ sub upgrade {
'task_manager_pidfile' => 'No more used',
'archived_pidfile' => 'No more used',
'bounced_pidfile' => 'No more used',
'use_fast_cgi' => 'No longer used', # 6.2.25b deprecated
'use_fast_cgi' => 'No longer used', # 6.2.25b deprecated
);
## Set language of new file content
......@@ -1773,6 +1772,38 @@ sub upgrade {
_get_canonical_read_date($sdm, 'update_admin')
)
);
$log->syslog('notice', 'Upgrading user dumps of closed lists.');
# Upgrading user dumps of closed lists.
my $lists =
Sympa::List::get_lists('*',
filter => [status => 'closed|family_closed']);