Upgrade.pm 92.9 KB
Newer Older
1
2
3
# -*- indent-tabs-mode: nil; -*-
# vim:ft=perl:et:sw=4
# $Id$
4
5

# Sympa - SYsteme de Multi-Postage Automatique
6
7
8
9
#
# Copyright (c) 1997, 1998, 1999 Institut Pasteur & Christophe Wolfhugel
# Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
# 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
10
# Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
11
12
# Copyright 2017, 2018, 2019, 2020, 2021 The Sympa Community. See the
# AUTHORS.md file at the top-level directory of this distribution and at
13
# <https://github.com/sympa-community/sympa.git>.
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
26
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
27

28
package Sympa::Upgrade;
29
30

use strict;
31
use warnings;
32
use Cwd qw();
33
use Encode qw();
34
use English qw(-no_match_vars);
35
use File::Copy qw();
36
use MIME::Base64 qw();
37
use Time::Local qw();
38

39
use Sympa;
40
use Conf;
41
use Sympa::ConfDef;
42
use Sympa::Constants;
sikeda's avatar
sikeda committed
43
use Sympa::DatabaseManager;
44
use Sympa::Language;
45
use Sympa::List;
46
use Sympa::LockedFile;
47
use Sympa::Log;
48
use Sympa::Message;
49
use Sympa::Request;
50
use Sympa::Spool;
51
use Sympa::Spool::Archive;
52
use Sympa::Spool::Auth;
53
use Sympa::Spool::Digest;
54
use Sympa::Tools::Data;
55
56
use Sympa::Tools::File;
use Sympa::Tools::Text;
57

58
my $language = Sympa::Language->instance;
59
my $log      = Sympa::Log->instance;
60

61
62
## Return the previous Sympa version, ie the one listed in
## data_structure.version
63
sub get_previous_version {
64
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
65
    my $previous_version;
66

67
    if (-f $version_file) {
68
        unless (open VFILE, $version_file) {
69
            $log->syslog('err', 'Unable to open %s: %m', $version_file);
70
71
72
73
74
75
76
77
78
79
80
81
            return undef;
        }
        while (<VFILE>) {
            next if /^\s*$/;
            next if /^\s*\#/;
            chomp;
            $previous_version = $_;
            last;
        }
        close VFILE;

        return $previous_version;
82
    }
83

84
85
86
87
    return undef;
}

sub update_version {
88
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
89
90
91

    ## Saving current version if required
    unless (open VFILE, ">$version_file") {
92
        $log->syslog(
93
            'err',
94
            'Unable to write %s; sympa.pl needs write access on %s directory: %m',
95
            $version_file,
96
            $Conf::Conf{'etc'}
97
98
        );
        return undef;
99
    }
100
101
    printf VFILE
        "# This file is automatically created by sympa.pl after installation\n# Unless you know what you are doing, you should not modify it\n";
102
    printf VFILE "%s\n", Sympa::Constants::VERSION;
103
    close VFILE;
104

105
106
107
108
109
    return 1;
}

## Upgrade data structure from one version to another
sub upgrade {
110
    $log->syslog('debug3', '(%s, %s)', @_);
111
112
    my ($previous_version, $new_version) = @_;

113
    if (lower_version($new_version, $previous_version)) {
114
        $log->syslog('notice',
115
116
117
            'Installing  older version of Sympa ; no upgrade operation is required'
        );
        return 1;
118
119
    }

120
    ## Check database connectivity and probe database
sikeda's avatar
sikeda committed
121
    unless (Sympa::DatabaseManager::probe_db()) {
122
        $log->syslog(
123
            'err',
124
125
126
127
128
129
            'Database %s defined in sympa.conf has not the right structure or is unreachable. verify db_xxx parameters in sympa.conf',
            $Conf::Conf{'db_name'}
        );
        return undef;
    }

130
131
132
133
134
    # As of 6.2.33b.1, owners/moderators are no longer stored in config file.
    # - Write out initial permanent owners/editors in <role>.dump files.
    # - And, if list is not closed, import owners/moderators from those files
    #   into database.
    if (lower_version($previous_version, '6.2.33b.1')) {
IKEDA Soji's avatar
IKEDA Soji committed
135
136
137
        $log->syslog('notice',
            'Restoring users of ALL lists...it may take a while...');

138
        my $all_lists = Sympa::List::get_lists('*');
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        foreach my $list (@{$all_lists || []}) {
            next unless $list;
            my $dir = $list->{'dir'};

            my $fh;
            next unless open $fh, '<', $dir . '/config';
            my $config = do { local $RS; <$fh> };
            close $fh;

            $config =~ s/(\A|\n)[\t ]+(?=\n)/$1/g;    # normalize empty lines
            open my $ifh, '<', \$config;              # open "in memory" file
            my @config = do { local $RS = ''; <$ifh> };
            close $ifh;
            foreach my $role (qw(owner editor)) {
                my $file = $dir . '/' . $role . '.dump';
                if (!-e $file and open my $ofh, '>', $file) {
                    my $admins = join '', grep {/\A\s*$role\b/} @config;
                    print $ofh $admins;
                    close $ofh;
                }

                next
                    if $list->{'admin'}{'status'} eq 'closed'
                    or $list->{'admin'}{'status'} eq 'family_closed';
                $list->restore_users($role);
            }
        }
    }

IKEDA Soji's avatar
IKEDA Soji committed
168
169
    # Always update config.bin files while upgrading.
    # This is especially useful for character encoding reasons.
170
    $log->syslog('notice',
171
        'Rebuilding config.bin files for ALL lists...it may take a while...');
172
173
    my $all_lists = Sympa::List::get_lists('*', reload_config => 1);
    # Recreate admin_table entries. #FIXME: Is this needed here?
IKEDA Soji's avatar
IKEDA Soji committed
174
175
176
    $log->syslog('notice',
        'Rebuilding the admin_table...it may take a while...');
    foreach my $list (@{$all_lists || []}) {    # See GH #71
IKEDA Soji's avatar
IKEDA Soji committed
177
178
        $list->sync_include('owner');
        $list->sync_include('editor');
179
    }
180

181
    ## Migration to tt2
182
    if (lower_version($previous_version, '4.2b')) {
183

184
        $log->syslog('notice', 'Migrating templates to TT2 format...');
185
186
187

        my $tpl_script = Sympa::Constants::SCRIPTDIR . '/tpl2tt2.pl';
        unless (open EXEC, "$tpl_script|") {
188
            $log->syslog('err', 'Unable to run %s', $tpl_script);
189
190
191
192
            return undef;
        }
        close EXEC;

193
        $log->syslog('notice', 'Rebuilding web archives...');
194
        my $all_lists = Sympa::List::get_lists('*');
195
196
        foreach my $list (@$all_lists) {
            # FIXME: line below will always success
197
198
            next
                unless defined $list->{'admin'}{'web_archive'}
199
                or defined $list->{'admin'}{'archive'};
200

201
202
203
204
205
206
207
208
209
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
210
211
212
                next;
            }
        }
213
    }
214

215
    ## Initializing the new admin_table
216
    if (lower_version($previous_version, '4.2b.4')) {
217
        $log->syslog('notice', 'Initializing the new admin_table...');
218
        my $all_lists = Sympa::List::get_lists('*');
219
        foreach my $list (@$all_lists) {
IKEDA Soji's avatar
IKEDA Soji committed
220
221
            $list->sync_include('owner');
            $list->sync_include('editor');
222
        }
223
224
225
    }

    ## Move old-style web templates out of the include_path
226
    if (lower_version($previous_version, '5.0.1')) {
227
        $log->syslog('notice',
228
229
            'Old web templates HTML structure is not compliant with latest ones.'
        );
230
        $log->syslog('notice',
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
            'Moving old-style web templates out of the include_path...');

        my @directories;

        if (-d "$Conf::Conf{'etc'}/web_tt2") {
            push @directories, "$Conf::Conf{'etc'}/web_tt2";
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {

            if (-d "$Conf::Conf{'etc'}/$vr/web_tt2") {
                push @directories, "$Conf::Conf{'etc'}/$vr/web_tt2";
            }
        }

        ## Search in V. Robot Lists
248
        my $all_lists = Sympa::List::get_lists('*');
249
250
251
252
253
254
255
256
257
258
        foreach my $list (@$all_lists) {
            if (-d "$list->{'dir'}/web_tt2") {
                push @directories, "$list->{'dir'}/web_tt2";
            }
        }

        my @templates;

        foreach my $d (@directories) {
            unless (opendir DIR, $d) {
259
260
                printf STDERR "Error: Cannot read %s directory: %s", $d,
                    $ERRNO;
261
262
263
264
265
266
267
268
269
270
271
272
273
                next;
            }

            foreach my $tt2 (sort grep(/\.tt2$/, readdir DIR)) {
                push @templates, "$d/$tt2";
            }

            closedir DIR;
        }

        foreach my $tpl (@templates) {
            unless (rename $tpl, "$tpl.oldtemplate") {
                printf STDERR
274
275
                    "Error : failed to rename %s to %s.oldtemplate: %s\n",
                    $tpl, $tpl, $ERRNO;
276
277
                next;
            }
278

279
            $log->syslog('notice', 'File %s renamed %s',
280
281
282
                $tpl, "$tpl.oldtemplate");
        }
    }
283
284

    ## Clean buggy list config files
285
    if (lower_version($previous_version, '5.1b')) {
286
        $log->syslog('notice', 'Cleaning buggy list config files...');
287
        my $all_lists = Sympa::List::get_lists('*');
288
289
290
        foreach my $list (@$all_lists) {
            $list->save_config('listmaster@' . $list->{'domain'});
        }
291
292
293
    }

    ## Fix a bug in Sympa 5.1
294
    if (lower_version($previous_version, '5.1.2')) {
295
        $log->syslog('notice', 'Rename archives/log. files...');
296
        my $all_lists = Sympa::List::get_lists('*');
297
298
299
300
301
302
        foreach my $list (@$all_lists) {
            if (-f $list->{'dir'} . '/archives/log.') {
                rename $list->{'dir'} . '/archives/log.',
                    $list->{'dir'} . '/archives/log.00';
            }
        }
303
304
    }

305
    if (lower_version($previous_version, '5.2a.1')) {
306

307
        ## Fill the robot_subscriber and robot_admin fields in DB
308
        $log->syslog('notice',
309
310
311
312
            'Updating the new robot_subscriber and robot_admin  Db fields...'
        );

        foreach my $r (keys %{$Conf::Conf{'robots'}}) {
313
            my $all_lists = Sympa::List::get_lists($r);
314
            foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
315
                my $sdm = Sympa::DatabaseManager->instance;
316
317
                foreach my $table ('subscriber', 'admin') {
                    unless (
sikeda's avatar
sikeda committed
318
319
                        $sdm
                        and $sdm->do_query(
320
321
322
                            "UPDATE %s_table SET robot_%s=%s WHERE (list_%s=%s)",
                            $table,
                            $table,
sikeda's avatar
sikeda committed
323
                            $sdm->quote($r),
324
                            $table,
sikeda's avatar
sikeda committed
325
                            $sdm->quote($list->{'name'})
326
                        )
Luc Didry's avatar
Luc Didry committed
327
                    ) {
328
                        $log->syslog(
329
                            'err',
330
                            'Unable to fille the robot_admin and robot_subscriber fields in database for robot %s',
331
332
                            $r
                        );
333
                        Sympa::send_notify_to_listmaster('*',
sikeda's avatar
sikeda committed
334
                            'upgrade_failed', {'error' => $sdm->error});
335
336
337
338
339
340
341
                        return undef;
                    }
                }
            }
        }

        ## Rename web archive directories using 'domain' instead of 'host'
342
        $log->syslog('notice',
343
344
345
346
347
            'Renaming web archive directories with the list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'arc_path');
        unless (opendir ARCDIR, $root_dir) {
348
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
349
350
351
352
353
354
355
356
357
358
359
360
            return undef;
        }

        foreach my $dir (sort readdir(ARCDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            my ($listname, $listdomain) = split /\@/, $dir;

            next unless ($listname && $listdomain);

361
            my $list = Sympa::List->new($listname, $listdomain);
362
            unless (defined $list) {
363
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
364
365
366
367
368
369
370
371
372
373
                next;
            }

            if ($listdomain ne $list->{'domain'}) {
                my $old_path =
                    $root_dir . '/' . $listname . '@' . $listdomain;
                my $new_path =
                    $root_dir . '/' . $listname . '@' . $list->{'domain'};

                if (-d $new_path) {
374
                    $log->syslog(
375
                        'err',
376
                        'Could not rename %s to %s; directory already exists',
377
378
379
380
381
382
                        $old_path,
                        $new_path
                    );
                    next;
                } else {
                    unless (rename $old_path, $new_path) {
383
                        $log->syslog('err', 'Failed to rename %s to %s: %m',
384
                            $old_path, $new_path);
385
386
                        next;
                    }
387
                    $log->syslog('notice', "Renamed %s to %s",
388
389
390
391
392
393
                        $old_path, $new_path);
                }
            }
        }
        close ARCDIR;

394
395
396
    }

    ## DB fields of enum type have been changed to int
397
    if (lower_version($previous_version, '5.2a.1')) {
398
        if ($Conf::Conf{'db_type'} eq 'mysql') {
399
400
401
402
403
404
405
406
407
408
409
            my %check = (
                'subscribed_subscriber' => 'subscriber_table',
                'included_subscriber'   => 'subscriber_table',
                'subscribed_admin'      => 'admin_table',
                'included_admin'        => 'admin_table'
            );

            foreach my $field (keys %check) {
                my $statement;
                my $sth;

sikeda's avatar
sikeda committed
410
411
412
413
414
415
416
                my $sdm = Sympa::DatabaseManager->instance;
                unless (
                    $sdm
                    and $sth = $sdm->do_query(
                        q{SELECT max(%s) FROM %s},
                        $field, $check{$field}
                    )
Luc Didry's avatar
Luc Didry committed
417
                ) {
418
                    $log->syslog('err', 'Unable to prepare SQL statement');
419
420
421
422
423
424
425
426
427
428
                    return undef;
                }

                my $max = $sth->fetchrow();
                $sth->finish();

                ## '0' has been mapped to 1 and '1' to 2
                ## Restore correct field value
                if ($max > 1) {
                    ## 1 to 0
429
                    $log->syslog('notice',
430
                        'Fixing DB field %s; turning 1 to 0...', $field);
431
432
                    my $rows;
                    $sth =
sikeda's avatar
sikeda committed
433
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
434
435
                        $check{$field}, $field, 0, $field, 1);
                    unless ($sth) {
436
437
                        $log->syslog('err',
                            'Unable to execute SQL statement');
438
439
440
                        return undef;
                    }
                    $rows = $sth->rows;
441
                    $log->syslog('notice', 'Updated %d rows', $rows);
442
443

                    ## 2 to 1
444
                    $log->syslog('notice',
445
                        'Fixing DB field %s; turning 2 to 1...', $field);
446
447

                    $sth =
sikeda's avatar
sikeda committed
448
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
449
450
                        $check{$field}, $field, 1, $field, 2);
                    unless ($sth) {
451
452
                        $log->syslog('err',
                            'Unable to execute SQL statement');
453
454
455
                        return undef;
                    }
                    $rows = $sth->rows;
456
                    $log->syslog('notice', 'Updated %d rows', $rows);
457
458
459
460
                }

                ## Set 'subscribed' data field to '1' is none of 'subscribed'
                ## and 'included' is set
461
                $log->syslog('notice',
462
463
                    'Updating subscribed field of the subscriber table...');
                my $rows;
sikeda's avatar
sikeda committed
464
                $sth = $sdm->do_query(
465
                    q{UPDATE subscriber_table
466
467
468
469
470
                      SET subscribed_subscriber = 1
                      WHERE (included_subscriber IS NULL OR
                             included_subscriber <> 1) AND
                            (subscribed_subscriber IS NULL OR
                             subscribed_subscriber <> 1)}
471
472
                );
                unless ($sth) {
473
                    $log->syslog('err', 'Unable to execute SQL statement');
474
475
476
                    return undef;
                }
                $rows = $sth->rows;
477
                $log->syslog('notice', '%d rows have been updated', $rows);
478
479
            }
        }
480
481
482
    }

    ## Rename bounce sub-directories
483
    if (lower_version($previous_version, '5.2a.1')) {
484

485
        $log->syslog('notice',
486
487
488
489
490
            'Renaming bounce sub-directories adding list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'bounce_path');
        unless (opendir BOUNCEDIR, $root_dir) {
491
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
492
493
494
495
496
497
498
499
500
501
502
503
504
            return undef;
        }

        foreach my $dir (sort readdir(BOUNCEDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            ## Directory already include the list domain
            next
                if ($dir =~ /\@/);

            my $listname = $dir;
505
            my $list     = Sympa::List->new($listname);
506
            unless (defined $list) {
507
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
508
509
510
511
512
513
514
515
                next;
            }

            my $old_path = $root_dir . '/' . $listname;
            my $new_path =
                $root_dir . '/' . $listname . '@' . $list->{'domain'};

            if (-d $new_path) {
516
                $log->syslog('err',
517
                    'Could not rename %s to %s; directory already exists',
518
519
520
521
                    $old_path, $new_path);
                next;
            } else {
                unless (rename $old_path, $new_path) {
522
                    $log->syslog('err', 'Failed to rename %s to %s: %m',
523
                        $old_path, $new_path);
524
525
                    next;
                }
526
                $log->syslog('notice', "Renamed %s to %s",
527
528
529
530
                    $old_path, $new_path);
            }
        }
        close BOUNCEDIR;
531
532
    }

533
    # Update lists config using 'include_sympa_list'
534
    if (lower_version($previous_version, '5.2a.1')) {
535
        $log->syslog('notice',
536
            'Update lists config using include_sympa_list parameter...');
537

538
        my $all_lists = Sympa::List::get_lists('*');
539
        foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
540
541
542
543
544
            my @include_lists =
                @{$list->{'admin'}{'include_sympa_list'} || []};
            my $changed = 0;
            foreach my $incl (@include_lists) {
                # Search for the list if robot is not specified.
545
546
                my $incl_list =
                    Sympa::List->new($incl->{listname}, $list->{'domain'});
sikeda's avatar
sikeda committed
547
548
549
550
551
552
553
554

                if (    $incl_list
                    and $incl_list->{'domain'} ne $list->{'domain'}) {
                    $log->syslog('notice',
                        'Update config file of list %s, including list %s',
                        $list->get_id, $incl_list->get_id);
                    $incl->{listname} = $incl_list->get_id;
                    $changed = 1;
555
                }
sikeda's avatar
sikeda committed
556
557
558
559
560
            }
            if ($changed) {
                $list->{'admin'}{'include_sympa_list'} = [@include_lists];
                $list->save_config(Sympa::get_address($list, 'listmaster'));
            }
561
        }
562
563
564
    }

    ## New mhonarc ressource file with utf-8 recoding
565
    if (lower_version($previous_version, '5.3a.6')) {
566

567
        $log->syslog('notice',
568
569
570
571
572
573
574
575
576
577
578
579
            'Looking for customized mhonarc-ressources.tt2 files...');
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            my $etc_dir = $Conf::Conf{'etc'};

            if ($vr ne $Conf::Conf{'domain'}) {
                $etc_dir .= '/' . $vr;
            }

            if (-f $etc_dir . '/mhonarc-ressources.tt2') {
                my $new_filename =
                    $etc_dir . '/mhonarc-ressources.tt2' . '.' . time;
                rename $etc_dir . '/mhonarc-ressources.tt2', $new_filename;
580
                $log->syslog(
581
582
583
584
585
                    'notice',
                    "Custom %s file has been backed up as %s",
                    $etc_dir . '/mhonarc-ressources.tt2',
                    $new_filename
                );
586
                Sympa::send_notify_to_listmaster('*', 'file_removed',
587
588
589
590
                    [$etc_dir . '/mhonarc-ressources.tt2', $new_filename]);
            }
        }

591
        $log->syslog('notice', 'Rebuilding web archives...');
592
        my $all_lists = Sympa::List::get_lists('*');
593
594
        foreach my $list (@$all_lists) {
            # FIXME: next line always success
595
596
            next
                unless defined $list->{'admin'}{'web_archive'}
597
                or defined $list->{'admin'}{'archive'};
598

599
600
601
602
603
604
605
606
607
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
608
609
610
                next;
            }
        }
611
612
613
614

    }

    ## Changed shared documents name encoding
615
616
    ## They are Q-encoded therefore easier to store on any filesystem with any
    ## encoding
617
    if (lower_version($previous_version, '5.3a.8')) {
618
        $log->syslog('notice', 'Q-Encoding web documents filenames...');
619
620
621
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists';
622
    }
623

624
625
    ## We now support UTF-8 only for custom templates, config files, headers
    ## and footers, info files
626
    ## + web_tt2, scenari, create_list_templatee, families
627
    if (lower_version($previous_version, '5.3b.3')) {
628
        $log->syslog('notice', 'Encoding all custom files to UTF-8...');
629
630
631
632
633
634
635
636

        my (@directories, @files);

        ## Site level
        foreach my $type (
            'mail_tt2', 'web_tt2',
            'scenari',  'create_list_templates',
            'families'
Luc Didry's avatar
Luc Didry committed
637
        ) {
638
639
640
641
642
643
644
645
646
647
648
            if (-d $Conf::Conf{'etc'} . '/' . $type) {
                push @directories,
                    [$Conf::Conf{'etc'} . '/' . $type, $Conf::Conf{'lang'}];
            }
        }

        foreach my $f (
            Conf::get_sympa_conf(),
            Conf::get_wwsympa_conf(),
            $Conf::Conf{'etc'} . '/' . 'topics.conf',
            $Conf::Conf{'etc'} . '/' . 'auth.conf'
Luc Didry's avatar
Luc Didry committed
649
        ) {
650
651
652
653
654
655
656
657
658
659
660
            if (-f $f) {
                push @files, [$f, $Conf::Conf{'lang'}];
            }
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            foreach my $type (
                'mail_tt2', 'web_tt2',
                'scenari',  'create_list_templates',
                'families'
Luc Didry's avatar
Luc Didry committed
661
            ) {
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
                if (-d $Conf::Conf{'etc'} . '/' . $vr . '/' . $type) {
                    push @directories,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $type,
                        Conf::get_robot_conf($vr, 'lang')
                        ];
                }
            }

            foreach my $f ('robot.conf', 'topics.conf', 'auth.conf') {
                if (-f $Conf::Conf{'etc'} . '/' . $vr . '/' . $f) {
                    push @files,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $f,
                        $Conf::Conf{'lang'}
                        ];
                }
            }
        }

        ## Search in Lists
683
        my $all_lists = Sympa::List::get_lists('*');
684
685
686
687
688
        foreach my $list (@$all_lists) {
            foreach my $f (
                'config',   'info',
                'homepage', 'message.header',
                'message.footer'
Luc Didry's avatar
Luc Didry committed
689
            ) {
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
                if (-f $list->{'dir'} . '/' . $f) {
                    push @files,
                        [$list->{'dir'} . '/' . $f, $list->{'admin'}{'lang'}];
                }
            }

            foreach my $type ('mail_tt2', 'web_tt2', 'scenari') {
                my $directory = $list->{'dir'} . '/' . $type;
                if (-d $directory) {
                    push @directories, [$directory, $list->{'admin'}{'lang'}];
                }
            }
        }

        ## Search language directories
        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }

            if ($d =~ /(mail_tt2|web_tt2)$/) {
                foreach
                    my $subdir (grep(/^[a-z]{2}(_[A-Z]{2})?$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories, ["$d/$subdir", $subdir];
                    }
                }
                closedir DIR;

            } elsif ($d =~ /(create_list_templates|families)$/) {
                foreach my $subdir (grep(/^\w+$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories,
                            ["$d/$subdir", $Conf::Conf{'lang'}];
                    }
                }
                closedir DIR;
            }
        }

        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }
            foreach my $file (readdir DIR) {
                next
                    unless (
                    (   $d =~
                        /mail_tt2|web_tt2|create_list_templates|families/
                        && $file =~ /\.tt2$/
                    )
                    || ($d =~ /scenari$/ && $file =~ /\w+\.\w+$/)
                    );
                push @files, [$d . '/' . $file, $lang];
            }
            closedir DIR;
        }

        ## Do the encoding modifications
        ## Previous versions of files are backed up with the date extension
        my $total = to_utf8(\@files);
753
        $log->syslog('notice', '%d files have been modified', $total);
754
755
    }

756
757
    ## giving up subscribers flat files ; moving subscribers to the DB
    ## Also giving up old 'database' mode
758
    if (lower_version($previous_version, '5.4a.1')) {
759

760
        $log->syslog('notice',
761
762
763
            'Looking for lists with user_data_source parameter set to file or database...'
        );

764
        my $all_lists = Sympa::List::get_lists('*');
765
766
767
768
        foreach my $list (@$all_lists) {

            if ($list->{'admin'}{'user_data_source'} eq 'file') {

769
                $log->syslog(
770
                    'notice',
771
                    'List %s; changing user_data_source from file to include2...',
772
773
774
                    $list->{'name'}
                );

775
776
777
778
                # Load <list dir>/subscribers to the DB
                if (-e $list->{'dir'} . '/subscribers'
                    and rename $list->{'dir'} . '/subscribers',
                    $list->{'dir'} . '/member.dump'
Luc Didry's avatar
Luc Didry committed
779
                ) {
780
                    $list->restore_users('member');
781

782
783
784
                    my $total = $list->{'add_outcome'}{'added_members'};
                    if (defined $list->{'add_outcome'}{'errors'}) {
                        $log->syslog('err', 'Failed to add users: %s',
Luc Didry's avatar
Luc Didry committed
785
786
                            $list->{'add_outcome'}{'errors'}{'error_message'}
                        );
787
788
789
790
                    }
                    $log->syslog('notice',
                        '%d subscribers have been loaded into the database',
                        $total);
791
792
                }

793
                $list->{'admin'}{'user_data_source'} = 'include2';
794
795

                unless ($list->save_config('automatic')) {
796
                    $log->syslog('err',
797
                        'Failed to save config file for list %s', $list);
798
799
800
                }
            } elsif ($list->{'admin'}{'user_data_source'} eq 'database') {

801
                $log->syslog(
802
                    'notice',
803
                    'List %s; changing user_data_source from database to include2...',
804
805
806
807
                    $list->{'name'}
                );

                unless ($list->update_list_member('*', {'subscribed' => 1})) {
808
                    $log->syslog('err',
809
810
811
812
813
814
                        'Failed to update subscribed DB field');
                }

                $list->{'admin'}{'user_data_source'} = 'include2';

                unless ($list->save_config('automatic')) {
815
                    $log->syslog('err',
816
817
818
819
820
                        'Failed to save config file for list %s',
                        $list->{'name'});
                }
            }
        }
821
    }
822

823
    if (lower_version($previous_version, '5.5a.1')) {
sikeda's avatar
sikeda committed
824
        # Remove OTHER/ subdirectories in bounces
825
        $log->syslog('notice', "Removing obsolete OTHER/ bounce directories");
sikeda's avatar
sikeda committed
826
827
        if (opendir my $dh, $Conf::Conf{'bounce_path'}) {
            foreach my $subdir (sort grep (!/^\.+$/, readdir $dh)) {
828
                my $other_dir =
sikeda's avatar
sikeda committed
829
                    $Conf::Conf{'bounce_path'} . '/' . $subdir . '/OTHER';
830
                if (-d $other_dir) {
831
                    Sympa::Tools::File::remove_dir($other_dir);
832
833
                    $log->syslog('notice', 'Directory %s removed',
                        $other_dir);
834
835
                }
            }
sikeda's avatar
sikeda committed
836
            closedir $dh;
837
        } else {
838
            $log->syslog(
839
840
                'err',
                'Failed to open directory %s: %m',
sikeda's avatar
sikeda committed
841
                $Conf::Conf{'bounce_path'}
842
            );
843
844
845
        }
    }

846
    if (lower_version($previous_version, '6.1b.5')) {
847
848
849
850
851
        ## Encoding of shared documents was not consistent with recent
        ## versions of MIME::Encode
        ## MIME::EncWords::encode_mimewords() used to encode characters -!*+/
        ## Now these characters are preserved, according to RFC 2047 section 5
        ## We change encoding of shared documents according to new algorithm
852
        $log->syslog('notice',
853
            'Fixing Q-encoding of web document filenames...');
854
855
856
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists', '--fix_qencode';
857
    }
858
    if (lower_version($previous_version, '6.1.11')) {
859
        ## Exclusion table was not robot-enabled.
860
        $log->syslog('notice', 'Fixing robot column of exclusion table');
861
        my $sth;
sikeda's avatar
sikeda committed
862
        my $sdm = Sympa::DatabaseManager->instance;
863
864
865
866
867
868
869
        unless (
            $sdm and $sth = $sdm->do_query(
                q{SELECT *
                  FROM exclusion_table
                  WHERE robot_exclusion IS NULL OR robot_exclusion = ''}
            )
        ) {
870
            $log->syslog('err',
871
                'Unable to gather information from the exclusions table');
872
        }
873
874
875
876
        my $rows = $sth->fetchall_arrayref(
            {list_exclusion => 1, user_exclusion => 1});
        $sth->finish;

877
        my @robots = Sympa::List::get_robots();
878
879
        foreach my $data (@{$rows || []}) {
            # Guessing right robot for each exclusion.
880
881
            my @valid_robot_candidates;
            foreach my $robot (@robots) {
882
883
884
885
                next
                    unless Sympa::List->new($data->{'list_exclusion'},
                    $robot, {just_try => 1});
                push @valid_robot_candidates, $robot;
886
            }
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
            unless (@valid_robot_candidates) {
                unless (
                    $sdm->do_prepared_query(
                        q{DELETE FROM exclusion_table
                          WHERE (robot_exclusion IS NULL OR
                                 robot_exclusion = '') AND
                                list_exclusion = ? AND user_exclusion = ?},
                        $data->{list_exclusion}, $data->{user_exclusion}
                    )
                ) {
                    $log->syslog(
                        'err',
                        'Unable to delete entry (%s, %s) in exclusions table',
                        $data->{'list_exclusion'},
                        $data->{'user_exclusion'}
                    );
                }
            } elsif (1 == scalar @valid_robot_candidates) {
                my $valid_robot = $valid_robot_candidates[0];
906
                unless (
907
908
909
910
911
912
913
914
                    $sdm->do_prepared_query(
                        q{UPDATE exclusion_table
                          SET robot_exclusion = ?
                          WHERE (robot_exclusion IS NULL OR
                                 robot_exclusion = '') AND
                                list_exclusion = ? AND user_exclusion = ?},
                        $valid_robot,
                        $data->{'list_exclusion'}, $data->{'user_exclusion'}
915
                    )
Luc Didry's avatar
Luc Didry committed
916
                ) {
917
                    $log->syslog(
918
                        'err',
919
                        'Unable to update entry (%s, %s) in exclusions table (trying to add robot %s)',
920
921
922
923
924
925
                        $data->{'list_exclusion'},
                        $data->{'user_exclusion'},
                        $valid_robot
                    );
                }
            } else {
926
                $log->syslog(
927
                    'err',
928
                    'Exclusion robot could not be guessed for user "%s" in list "%s". Either this user is no longer subscribed to the list or the list appears in more than one robot (or the query to the database failed). Here is the list of robots in which this list name appears: "%s"',
929
930
                    $data->{'user_exclusion'},
                    $data->{'list_exclusion'},
931
                    join(', ', @valid_robot_candidates)
932
933
934
935
                );
            }
        }
        ## Caching all lists config subset to database
936
        $log->syslog('notice', 'Caching all list config to database...');
937
938
        Sympa::List::_flush_list_db();
        my $all_lists = Sympa::List::get_lists('*', 'reload_config' => 1);
939
940
941
        foreach my $list (@$all_lists) {
            $list->_update_list_db;
        }
942
        $log->syslog('notice', '...done');
943
    }
944

945
    ## We have obsoleted wwsympa.conf.  It would be migrated to sympa.conf.
946
    if (lower_version($previous_version, '6.2b.1')) {
947
948
        $log->syslog('notice', 'Migrating wwsympa.conf...');

949
950
951
952
953
        my $sympa_conf   = Conf::get_sympa_conf();
        my $wwsympa_conf = Conf::get_wwsympa_conf();
        my $fh;
        my %migrated = ();
        my @newconf  = ();
954
        my ($date, $human_date);
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991

        ## Some sympa.conf parameters were overridden by wwsympa.conf.
        ## Others prefer sympa.conf.
        my %wwsconf_override = (
            'arc_path'                   => 'yes',
            'archive_default_index'      => 'yes',
            'bounce_path'                => 'yes',
            'cookie_domain'              => 'NO',
            'cookie_expire'              => 'yes',
            'cookie_refresh'             => 'NO',
            'custom_archiver'            => 'yes',
            'default_home'               => 'NO',
            'export_topics'              => 'yes',
            'html_editor_file'           => 'NO',    # 6.2a
            'html_editor_init'           => 'NO',
            'ldap_force_canonical_email' => 'NO',
            'log_facility'               => 'yes',
            'mhonarc'                    => 'yes',
            'password_case'              => 'NO',
            'review_page_size'           => 'yes',
            'title'                      => 'NO',
            'use_html_editor'            => 'NO',
            'viewlogs_page_size'         => 'yes',
            'wws_path'                   => undef,
        );
        ## Old params
        my %old_param = (
            'alias_manager' => 'No more used, using '
                . $Conf::Conf{'alias_manager'},
            'wws_path' => 'No more used',
            'icons_url' =>
                'No more used. Using static_content/icons instead.',
            'robots' =>
                'Not used anymore. Robots are fully described in their respective robot.conf file.',
            'task_manager_pidfile' => 'No more used',
            'archived_pidfile'     => 'No more used',
            'bounced_pidfile'      => 'No more used',
992
            'use_fast_cgi'         => 'No longer used',   # 6.2.25b deprecated
IKEDA Soji's avatar
IKEDA Soji committed
993
            'htmlarea_url'         => 'No longer used',   # 6.2.36 deprecated
994
995
996
997
        );

        ## Set language of new file content
        $language->push_lang($Conf::Conf{'lang'});
Sympa authors's avatar
Sympa authors committed
998
        $date       = time;
999
1000
        $human_date = $language->gettext_strftime('%d %b %Y at %H:%M:%S',
            localtime $date);
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016

        if (-r $wwsympa_conf) {
            ## load only sympa.conf
            my $conf = (
                Conf::_load_config_file_to_hash(
                    {'path_to_config_file' => $sympa_conf}
                    )
                    || {}
            )->{'config'};
            # not yet implemented.
            #my $conf = Conf::load_robot_conf(
            #    {'robot' => '*', 'no_db' => 1, 'return_result' => 1}
            #);

            my %infile = ();
            ## load defaults
1017
            foreach my $p (@Sympa::ConfDef::params) {
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
                next unless $p->{'name'};
                next unless $p->{'file'};
                next unless $p->{'file'} eq 'wwsympa.conf';
                $infile{$p->{'name'}} = $p->{'default'};
            }
            ## get content of wwsympa.conf
            open my $fh, '<', $wwsympa_conf;
            while (<$fh>) {
                next if /^\s*#/;
                chomp $_;
                next unless /^\s*(\S+)\s+(.+)$/i;
                my ($k, $v) = ($1, $2);
                $infile{$k} = $v;
            }
            close $fh;

            my $name;
1035
            foreach my $p (@Sympa::ConfDef::params) {
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
                next unless $p->{'name'};
                $name = $p->{'name'};
                next unless exists $infile{$name};

                unless ($p->{'file'} and $p->{'file'} eq 'wwsympa.conf') {
                    ## may it exist in wwsympa.conf?
                    $migrated{'unknown'} ||= {};
                    $migrated{'unknown'}->{$name} = [$p, $infile{$name}];
                } elsif (exists $conf->{$name}) {
                    if ($wwsconf_override{$name} eq 'yes') {
                        ## does it override sympa.conf?
                        $migrated{'override'} ||= {};
                        $migrated{'override'}->{$name} = [