Merge branch 'sympa-6.2' into sync-performance

NEWS.md

 # Change Log
 
+## [6.2.63b.1](https://github.com/sympa-community/sympa/tree/6.2.63b.1) (2021-05-31)
+
+[Full Changelog](https://github.com/sympa-community/sympa/compare/6.2.62...6.2.63b.1)
+
+**Implemented enhancements:**
+
+- Hide archive download link from users without proper permissions [\#1176](https://github.com/sympa-community/sympa/pull/1176) [\#1179](https://github.com/sympa-community/sympa/pull/1179)
+- Obsolete `escape_chars()` / `unescape_chars()` [\#1167](https://github.com/sympa-community/sympa/pull/1167)
+- Use web URL in instruction if web interface is available [\#1159](https://github.com/sympa-community/sympa/pull/1159)
+- Add list visibility scenario for logged in users. [\#1140](https://github.com/sympa-community/sympa/pull/1140)
+- Add ability to forbid some list names [\#672](https://github.com/sympa-community/sympa/issues/672)
+
+**Fixed bugs:**
+
+- `bulk.pl` don't deliver mails if failed to personalize [\#1174](https://github.com/sympa-community/sympa/issues/1174)
+- Newlines in header fields added by ARC feature should be normalized [\#1168](https://github.com/sympa-community/sympa/issues/1168)
+- Error emails to listmaster from repeated `set_netidtoemail_db` calls [\#1163](https://github.com/sympa-community/sympa/issues/1163)
+- Sympa should not lock out users using password authentication with LDAP [\#1132](https://github.com/sympa-community/sympa/issues/1132)
+- `mailto:` link cannot be detected by some MUAs [\#1124](https://github.com/sympa-community/sympa/issues/1124)
+- Comments in HTTP Accept-Language field should be omitted [\#1152](https://github.com/sympa-community/sympa/pull/1152)
+- Empty paragraph in list config should not be warned. [\#1150](https://github.com/sympa-community/sympa/pull/1150)
+
+**Closed issues:**
+
+- Support scripts for maintenance [\#538](https://github.com/sympa-community/sympa/issues/538)
+
 ## [6.2.62](https://github.com/sympa-community/sympa/tree/6.2.62) (2021-04-27)
 
 [Full Changelog](https://github.com/sympa-community/sympa/compare/6.2.61b.2...6.2.62)

configure.ac

@@ -25,7 +25,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 AC_PREREQ(2.60)
-AC_INIT(sympa, 6.2.62, sympa-developpers@listes.renater.fr)
+AC_INIT(sympa, 6.2.63b.1, sympa-developpers@listes.renater.fr)
 AM_INIT_AUTOMAKE([foreign -Wall -Werror 1.9 tar-pax])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AM_PO_SUBDIRS

default/mail_tt2/report.tt2

@@ -46,6 +46,9 @@
 [%~ ELSIF report_entry == 'create_list_listmaster' ~%]
 [%|loc()%]List creation is restricted to listmaster.[%END%]
 
+[%~ ELSIF report_entry == 'create_list_closed' ~%]
+[%|loc()%]List creation is not allowed.[%END%]
+
 [%~ ELSIF report_entry == 'd_edit_subscriber' ~%]
 [%|loc()%]Shared document edition is restricted to list subscribers.[%END%]
 [%|loc()%]If you are subscribed to the list with a different email address, you should either use that other email address or update your list membership with the new email address.[%END%]

default/scenari/create_list.closed

+title.gettext closed
+
+true()   smtp,dkim,md5,smime -> reject('create_list_closed')
+

default/web_tt2/info.tt2

@@ -134,13 +134,14 @@
                             <li>
                                 <i class="fa-li fa fa-arrow-right"></i><a href="[% 'edit_list_request' | url_rel([list,'archives']) %]">[%|loc%]Change settings for who can view archives[%END%]</a>
                             </li>
+                            [% IF arc_access %]
                             <li>
                                 <i class="fa-li fa fa-arrow-right"></i><a href="[% 'arc_manage' | url_rel([list]) %]">[%|loc%]Download archives[%END%]</a>
                             </li>
+                            [% END %]
                         </ul>
                     </div>
                 </div>
-
                 <div class="item">
                     <div class="item_content">
                         <a class="item_title" href="[% 'edit_list_request' | url_rel([list,'data_source']) %]">

default/web_tt2/my.tt2

@@ -53,7 +53,7 @@
                                 <li><a href="[% 'review' | url_rel([l.key]) %]"><i class="fa fa-users"></i> [%|loc%]Review members[%END%]</a></li>
                             [% END %]
 
-                            [% IF is_user_allowed_to('archive_web_access', l.key) %]
+                            [% IF l.value.arc_access %]
                                 <li><a href="[% 'arc' | url_rel([l.key]) %]"><i class="fa fa-archive"></i> [%|loc%]Archives[%END%]</a></li>
                             [% END %]
                         </ul>

default/web_tt2/suspend_request.tt2

@@ -11,9 +11,9 @@
         <p>[%|loc%]You are subscribed to the following lists[%END%]</p>
 
         <form class="noborder toggleContainer" data-toggle-selector="input[name='listname']" action="[% path_cgi %]" method="POST" name="suspend_request">
-            [% IF which_info.size %]
+            [% IF which.size %]
                 <div class="item_list">
-                    [% FOREACH l = which_info %]
+                    [% FOREACH l = which %]
                         [% suspended = 0 %]
                         [% suspendable = 0 %]
                         [% additional_class = '' %]
@@ -63,7 +63,7 @@
                                             </a>
                                         </li>
                                     [% END %]
-                                    [% IF is_user_allowed_to('archive_web_access', l.key) %]
+                                    [% IF l.value.arc_access %]
                                         <li>
                                             <a href="[% 'arc' | url_rel([l.key]) %]">
                                                 [%|loc%]Archives[%END%]

src/bin/tpl2tt2.pl.in

@@ -9,6 +9,9 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
+# Copyright 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
+# <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/bin/upgrade_send_spool.pl.in

@@ -9,8 +9,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2017, 2019 The Sympa Community. See the AUTHORS.md file at
-# the top-level directory of this distribution and at
+# Copyright 2017, 2019, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/bin/upgrade_sympa_password.pl.in

@@ -9,8 +9,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2017, 2018, 2019 The Sympa Community. See the AUTHORS.md file at
-# the top-level directory of this distribution and at
+# Copyright 2017, 2018, 2019, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/cgi/wwsympa.fcgi.in

@@ -1619,30 +1619,8 @@ while ($query = Sympa::WWW::FastCGI->new) {
         $param->{'title_clear_txt'} = $param->{'title'};
     }
-    $param->{'is_user_allowed_to'} = sub {
-        my $function = shift;
-        my $list     = shift;
-        return 0 unless $function and $list;
-
-        $list = Sympa::List->new($list, $robot)
-            unless ref $list eq 'Sympa::List';
-
-        return 0
-            if $function eq 'subscribe'
-            and $param->{'user'}{'email'}
-            and $list->is_list_member($param->{'user'}{'email'});
-
-        my $result = Sympa::Scenario->new($list, $function)->authz(
-            $param->{'auth_method'},
-            {   'sender'      => $param->{'user'}{'email'},
-                'remote_host' => $param->{'remote_host'},
-                'remote_addr' => $param->{'remote_addr'}
-            }
-        );
-        return 0 unless ref $result eq 'HASH';
-        return 0 if $result->{action} =~ /\Areject\b/i;
-        return 1;
-    };
+    # Deprecated tt2 function. Compat. <= 6.2.62
+    $param->{'is_user_allowed_to'} = sub {0};
     ## store in session table this session contexte
     $session->store();
@@ -14184,9 +14162,10 @@ sub do_suspend_request {
 sub _set_my_lists_info {
     my $which = {};
-    # Set which_info unless in one list page
+    # Set 'which' unless in one list page
     if ($param->{'user'}{'email'} and ref $list ne 'Sympa::List') {
         my %get_which;
+        my %all_lists;
         foreach my $role (qw(member owner editor)) {
             $get_which{$role} = Sympa::List::get_lists(
@@ -14212,6 +14191,8 @@ sub _set_my_lists_info {
                 unless ref $result eq 'HASH'
                 and $result->{'action'} eq 'do_it';
+            $all_lists{$list->{'name'}} = $list;
+
             my $l = $list->{'name'};
             $which->{$l}{'subject'} = $list->{'admin'}{'subject'};
             $which->{$l}{'status'}  = $list->{'admin'}{'status'}; # new 6.2.46
@@ -14268,6 +14249,8 @@ sub _set_my_lists_info {
             $which->{$l}{'display'} = $which->{$l}{'listsuspend'};
         }
         foreach my $list (@{$get_which{owner}}) {
+            $all_lists{$list->{'name'}} = $list;
+
             my $l = $list->{'name'};
             $which->{$l}{'subject'} = $list->{'admin'}{'subject'};
@@ -14280,6 +14263,8 @@ sub _set_my_lists_info {
             $which->{$l}{'host'} = $list->{'domain'};
         }
         foreach my $list (@{$get_which{editor}}) {
+            $all_lists{$list->{'name'}} = $list;
+
             my $l = $list->{'name'};
             $which->{$l}{'subject'} = $list->{'admin'}{'subject'};
@@ -14291,6 +14276,25 @@ sub _set_my_lists_info {
             # Compat. < 6.2.32 (Not used by default)
             $which->{$l}{'host'} = $list->{'domain'};
         }
+
+        foreach my $list (values %all_lists) {
+            # Archives Access control
+            if (defined $list->is_archiving_enabled) {
+                my $result =
+                    Sympa::Scenario->new($list, 'archive_web_access')->authz(
+                    $param->{'auth_method'},
+                    {   'sender'      => $param->{'user'}{'email'},
+                        'remote_host' => $param->{'remote_host'},
+                        'remote_addr' => $param->{'remote_addr'}
+                    }
+                    );
+                my $r_action;
+                $r_action = $result->{'action'} if ref $result eq 'HASH';
+
+                $which->{$list->{'name'}}{arc_access} = 1
+                    if $r_action =~ /do_it/i;
+            }
+        }
     }
     $param->{'which'} = $which;

src/lib/Conf.pm

@@ -8,8 +8,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2017, 2018, 2019, 2020 The Sympa Community. See the AUTHORS.md
-# file at the top-level directory of this distribution and at
+# Copyright 2017, 2018, 2019, 2020, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/lib/Sympa/Aliases.pm

@@ -4,8 +4,8 @@
 
 # Sympa - SYsteme de Multi-Postage Automatique
 #
-# Copyright 2017, 2018 The Sympa Community. See the AUTHORS.md file at the
-# top-level directory of this distribution and at
+# Copyright 2017, 2018, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/lib/Sympa/Archive.pm

@@ -8,8 +8,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2018, 2019, 2020 The Sympa Community. See the AUTHORS.md
-# file at the top-level directory of this distribution and at
+# Copyright 2018, 2019, 2020, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/lib/Sympa/Config.pm

@@ -371,7 +371,7 @@ sub _sanitize_changes_set {
 
     return () unless ref $new eq 'ARRAY';    # Sanity check
     return () if $pitem->{obsolete};
-    return () unless $pitem->{privilege} eq 'write';
+    return () unless 'write' eq ($pitem->{privilege} // '');
 
     # Resolve synonym.
     if (ref $pitem->{synonym} eq 'HASH') {
@@ -425,7 +425,7 @@ sub _sanitize_changes_array {
 
     return () unless ref $new eq 'ARRAY';    # Sanity check
     return () if $pitem->{obsolete};
-    return () unless $pitem->{privilege} eq 'write';
+    return () unless 'write' eq ($pitem->{privilege} // '');
 
     my $i   = -1;
     my %ret = map {
@@ -475,7 +475,7 @@ sub _sanitize_changes_paragraph {
 
     return () unless ref $new eq 'HASH';    # Sanity check
     return () if $pitem->{obsolete};
-    return () unless $pitem->{privilege} eq 'write';
+    return () unless 'write' eq ($pitem->{privilege} // '');
 
     $self->_apply_defaults($cur, $pitem->{format},
         init => ($options{init} and not $options{loading}));
@@ -581,7 +581,7 @@ sub _sanitize_changes_leaf {
 
     return () if ref $new eq 'ARRAY';    # Sanity check: Hashref or scalar
     return () if $pitem->{obsolete};
-    return () unless $pitem->{privilege} eq 'write';
+    return () unless 'write' eq ($pitem->{privilege} // '');
 
     # If the parameter corresponds to a scenario or a task, mark it
     # as changed if its name was changed.  Example: 'subscribe'.

src/lib/Sympa/DatabaseDriver/PostgreSQL.pm

@@ -8,8 +8,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2018 The Sympa Community. See the AUTHORS.md file at the
-# top-level directory of this distribution and at
+# Copyright 2018, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -70,10 +70,9 @@ sub connect {
     $self->__dbh->do("SET DATESTYLE TO 'ISO';");
     $self->__dbh->do("SET NAMES 'utf8'");
     defined $self->__dbh->do(
-        q{CREATE TEMPORARY VIEW dual AS SELECT 'X'::varchar(1) AS dummy;}
-    ) or $self->__dbh->do(
-        q{CREATE OR REPLACE VIEW dual AS SELECT 'X'::varchar(1) AS dummy;}
-    );
+        q{CREATE TEMPORARY VIEW dual AS SELECT 'X'::varchar(1) AS dummy;})
+        or $self->__dbh->do(
+        q{CREATE OR REPLACE VIEW dual AS SELECT 'X'::varchar(1) AS dummy;});
 
     return 1;
 }

src/lib/Sympa/DatabaseDriver/SQLite.pm

@@ -8,8 +8,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2018 The Sympa Community. See the AUTHORS.md file at the
-# top-level directory of this distribution and at
+# Copyright 2018, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/lib/Sympa/HTMLDecorator.pm

@@ -8,6 +8,9 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
+# Copyright 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
+# <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

src/lib/Sympa/Language.pm

@@ -8,8 +8,8 @@
 # Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
 # 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
 # Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
-# Copyright 2017, 2018 The Sympa Community. See the AUTHORS.md file at the
-# top-level directory of this distribution and at
+# Copyright 2017, 2018, 2021 The Sympa Community. See the
+# AUTHORS.md file at the top-level directory of this distribution and at
 # <https://github.com/sympa-community/sympa.git>.
 #
 # This program is free software; you can redistribute it and/or modify

src/lib/Sympa/List.pm

@@ -1532,6 +1532,7 @@ sub send_probe_to_user {
 ## $list->delete_list_member('users' => \@u, 'exclude' => 1)
 ## $list->delete_list_member('users' => [$email], 'exclude' => 1)
 sub delete_list_member {
+    $log->syslog('debug2', '(%s, ...)', @_);
     my $self    = shift;
     my %param   = @_;
     my @u       = @{$param{'users'}};
@@ -1543,13 +1544,13 @@ sub delete_list_member {
 
     $log->syslog('debug2', '');
 
-    my $name  = $self->{'name'};
     my $total = 0;
 
     my $sdm = Sympa::DatabaseManager->instance;
     $sdm->begin;
 
     foreach my $who (@u) {
+        next unless defined $who and length $who;
         $who = Sympa::Tools::Text::canonic_email($who);
 
         ## Include in exclusion_table only if option is set.
@@ -1565,13 +1566,16 @@ sub delete_list_member {
                 q{DELETE FROM subscriber_table
                   WHERE user_subscriber = ? AND
                         list_subscriber = ? AND robot_subscriber = ?},
-                $who, $name, $self->{'domain'}
+                $who, $self->{'name'}, $self->{'domain'}
             )
         ) {
             $log->syslog('err', 'Unable to remove list member %s', $who);
             next;
         }
 
+        # Delete the pictures if any.
+        $self->delete_list_member_picture($who);
+
         # Delete signoff requests if any.
         my $spool_req = Sympa::Spool::Auth->new(
             context => $self,
@@ -1590,7 +1594,7 @@ sub delete_list_member {
         if ($operation) {
             $log->add_stat(
                 'robot'     => $self->{'domain'},
-                'list'      => $name,
+                'list'      => $self->{'name'},
                 'operation' => $operation,
                 'mail'      => $who
             );
@@ -1600,7 +1604,6 @@ sub delete_list_member {
     }
 
     $self->_cache_publish_expiry('member');
-    delete_list_member_picture($self, shift(@u));
 
         my $rc = $sdm->commit;
         unless ($rc) {
@@ -1614,15 +1617,16 @@ sub delete_list_member {
 
 ## Delete the indicated admin users from the list.
 sub delete_list_admin {
-    my ($self, $role, @u) = @_;
-    $log->syslog('debug2', '', $role);
+    $log->syslog('debug2', '(%s, %s, ...)', @_);
+    my $self = shift;
+    my $role = shift;
+    my @u    = @_;
 
-    my $name  = $self->{'name'};
     my $total = 0;
 
     foreach my $who (@u) {
+        next unless defined $who and length $who;
         $who = Sympa::Tools::Text::canonic_email($who);
-        my $statement;
 
         my $sdm = Sympa::DatabaseManager->instance;
 
@@ -3304,7 +3308,7 @@ sub add_list_member {
             }
         }
 
-        #Log in stat_table to make staistics
+        #Log in stat_table to make statistics
         $log->add_stat(
             'robot'     => $self->{'domain'},
             'list'      => $self->{'name'},