Upgrade.pm 82.8 KB
Newer Older
1
2
3
# -*- indent-tabs-mode: nil; -*-
# vim:ft=perl:et:sw=4
# $Id$
4
5

# Sympa - SYsteme de Multi-Postage Automatique
6
7
8
9
#
# Copyright (c) 1997, 1998, 1999 Institut Pasteur & Christophe Wolfhugel
# Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
# 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
10
# Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
11
12
# Copyright 2017, 2018 The Sympa Community. See the AUTHORS.md file at the
# top-level directory of this distribution and at
13
# <https://github.com/sympa-community/sympa.git>.
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
26
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
27

28
package Sympa::Upgrade;
29
30

use strict;
31
use warnings;
32
use Cwd qw();
33
use Encode qw();
34
use English qw(-no_match_vars);
35
use MIME::Base64 qw();
36
use Time::Local qw();
37

38
use Sympa;
39
use Conf;
40
use Sympa::ConfDef;
41
use Sympa::Constants;
sikeda's avatar
sikeda committed
42
use Sympa::DatabaseManager;
43
use Sympa::Language;
44
use Sympa::List;
45
use Sympa::LockedFile;
46
use Sympa::Log;
47
use Sympa::Message;
48
use Sympa::Request;
IKEDA Soji's avatar
IKEDA Soji committed
49
use Sympa::Scenario;
50
use Sympa::Spool;
51
use Sympa::Spool::Archive;
52
use Sympa::Spool::Auth;
53
use Sympa::Spool::Digest;
54
use Sympa::Tools::Data;
55
56
use Sympa::Tools::File;
use Sympa::Tools::Text;
57

58
my $language = Sympa::Language->instance;
59
my $log      = Sympa::Log->instance;
60

61
62
## Return the previous Sympa version, ie the one listed in
## data_structure.version
63
sub get_previous_version {
64
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
65
    my $previous_version;
66

67
    if (-f $version_file) {
68
        unless (open VFILE, $version_file) {
69
            $log->syslog('err', 'Unable to open %s: %m', $version_file);
70
71
72
73
74
75
76
77
78
79
80
81
            return undef;
        }
        while (<VFILE>) {
            next if /^\s*$/;
            next if /^\s*\#/;
            chomp;
            $previous_version = $_;
            last;
        }
        close VFILE;

        return $previous_version;
82
    }
83

84
85
86
87
    return undef;
}

sub update_version {
88
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
89
90
91

    ## Saving current version if required
    unless (open VFILE, ">$version_file") {
92
        $log->syslog(
93
            'err',
94
            'Unable to write %s; sympa.pl needs write access on %s directory: %m',
95
            $version_file,
96
            $Conf::Conf{'etc'}
97
98
        );
        return undef;
99
    }
100
101
    printf VFILE
        "# This file is automatically created by sympa.pl after installation\n# Unless you know what you are doing, you should not modify it\n";
102
    printf VFILE "%s\n", Sympa::Constants::VERSION;
103
    close VFILE;
104

105
106
107
108
109
    return 1;
}

## Upgrade data structure from one version to another
sub upgrade {
110
    $log->syslog('debug3', '(%s, %s)', @_);
111
112
    my ($previous_version, $new_version) = @_;

113
    if (lower_version($new_version, $previous_version)) {
114
        $log->syslog('notice',
115
116
117
            'Installing  older version of Sympa ; no upgrade operation is required'
        );
        return 1;
118
119
    }

120
    ## Check database connectivity and probe database
sikeda's avatar
sikeda committed
121
    unless (Sympa::DatabaseManager::probe_db()) {
122
        $log->syslog(
123
            'err',
124
125
126
127
128
129
            'Database %s defined in sympa.conf has not the right structure or is unreachable. verify db_xxx parameters in sympa.conf',
            $Conf::Conf{'db_name'}
        );
        return undef;
    }

130
131
132
133
134
    # As of 6.2.33b.1, owners/moderators are no longer stored in config file.
    # - Write out initial permanent owners/editors in <role>.dump files.
    # - And, if list is not closed, import owners/moderators from those files
    #   into database.
    if (lower_version($previous_version, '6.2.33b.1')) {
IKEDA Soji's avatar
IKEDA Soji committed
135
136
137
        $log->syslog('notice',
            'Restoring users of ALL lists...it may take a while...');

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        my $all_lists = Sympa::List::get_lists('*', skip_sync_admin => 1);
        foreach my $list (@{$all_lists || []}) {
            next unless $list;
            my $dir = $list->{'dir'};

            my $fh;
            next unless open $fh, '<', $dir . '/config';
            my $config = do { local $RS; <$fh> };
            close $fh;

            $config =~ s/(\A|\n)[\t ]+(?=\n)/$1/g;    # normalize empty lines
            open my $ifh, '<', \$config;              # open "in memory" file
            my @config = do { local $RS = ''; <$ifh> };
            close $ifh;
            foreach my $role (qw(owner editor)) {
                my $file = $dir . '/' . $role . '.dump';
                if (!-e $file and open my $ofh, '>', $file) {
                    my $admins = join '', grep {/\A\s*$role\b/} @config;
                    print $ofh $admins;
                    close $ofh;
                }

                next
                    if $list->{'admin'}{'status'} eq 'closed'
                    or $list->{'admin'}{'status'} eq 'family_closed';
                $list->restore_users($role);
            }
        }
    }

IKEDA Soji's avatar
IKEDA Soji committed
168
169
    # Always update config.bin files while upgrading.
    # This is especially useful for character encoding reasons.
170
    $log->syslog('notice',
171
        'Rebuilding config.bin files for ALL lists...it may take a while...');
IKEDA Soji's avatar
IKEDA Soji committed
172
173
    my $all_lists =
        Sympa::List::get_lists('*', reload_config => 1, skip_sync_admin => 1);
IKEDA Soji's avatar
IKEDA Soji committed
174
175
176
177
178
    # Recreate admin_table entries.
    $log->syslog('notice',
        'Rebuilding the admin_table...it may take a while...');
    foreach my $list (@{$all_lists || []}) {    # See GH #71
        $list->sync_include_admin;
179
    }
180

181
    ## Migration to tt2
182
    if (lower_version($previous_version, '4.2b')) {
183

184
        $log->syslog('notice', 'Migrating templates to TT2 format...');
185
186
187

        my $tpl_script = Sympa::Constants::SCRIPTDIR . '/tpl2tt2.pl';
        unless (open EXEC, "$tpl_script|") {
188
            $log->syslog('err', 'Unable to run %s', $tpl_script);
189
190
191
192
            return undef;
        }
        close EXEC;

193
        $log->syslog('notice', 'Rebuilding web archives...');
194
        my $all_lists = Sympa::List::get_lists('*');
195
196
        foreach my $list (@$all_lists) {
            # FIXME: line below will always success
197
198
            next
                unless defined $list->{'admin'}{'web_archive'}
199
                or defined $list->{'admin'}{'archive'};
200

201
202
203
204
205
206
207
208
209
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
210
211
212
                next;
            }
        }
213
    }
214

215
    ## Initializing the new admin_table
216
    if (lower_version($previous_version, '4.2b.4')) {
217
        $log->syslog('notice', 'Initializing the new admin_table...');
218
        my $all_lists = Sympa::List::get_lists('*');
219
220
221
        foreach my $list (@$all_lists) {
            $list->sync_include_admin();
        }
222
223
224
    }

    ## Move old-style web templates out of the include_path
225
    if (lower_version($previous_version, '5.0.1')) {
226
        $log->syslog('notice',
227
228
            'Old web templates HTML structure is not compliant with latest ones.'
        );
229
        $log->syslog('notice',
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
            'Moving old-style web templates out of the include_path...');

        my @directories;

        if (-d "$Conf::Conf{'etc'}/web_tt2") {
            push @directories, "$Conf::Conf{'etc'}/web_tt2";
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {

            if (-d "$Conf::Conf{'etc'}/$vr/web_tt2") {
                push @directories, "$Conf::Conf{'etc'}/$vr/web_tt2";
            }
        }

        ## Search in V. Robot Lists
247
        my $all_lists = Sympa::List::get_lists('*');
248
249
250
251
252
253
254
255
256
257
        foreach my $list (@$all_lists) {
            if (-d "$list->{'dir'}/web_tt2") {
                push @directories, "$list->{'dir'}/web_tt2";
            }
        }

        my @templates;

        foreach my $d (@directories) {
            unless (opendir DIR, $d) {
258
259
                printf STDERR "Error: Cannot read %s directory: %s", $d,
                    $ERRNO;
260
261
262
263
264
265
266
267
268
269
270
271
272
                next;
            }

            foreach my $tt2 (sort grep(/\.tt2$/, readdir DIR)) {
                push @templates, "$d/$tt2";
            }

            closedir DIR;
        }

        foreach my $tpl (@templates) {
            unless (rename $tpl, "$tpl.oldtemplate") {
                printf STDERR
273
274
                    "Error : failed to rename %s to %s.oldtemplate: %s\n",
                    $tpl, $tpl, $ERRNO;
275
276
                next;
            }
277

278
            $log->syslog('notice', 'File %s renamed %s',
279
280
281
                $tpl, "$tpl.oldtemplate");
        }
    }
282
283

    ## Clean buggy list config files
284
    if (lower_version($previous_version, '5.1b')) {
285
        $log->syslog('notice', 'Cleaning buggy list config files...');
286
        my $all_lists = Sympa::List::get_lists('*');
287
288
289
        foreach my $list (@$all_lists) {
            $list->save_config('listmaster@' . $list->{'domain'});
        }
290
291
292
    }

    ## Fix a bug in Sympa 5.1
293
    if (lower_version($previous_version, '5.1.2')) {
294
        $log->syslog('notice', 'Rename archives/log. files...');
295
        my $all_lists = Sympa::List::get_lists('*');
296
297
298
299
300
301
        foreach my $list (@$all_lists) {
            if (-f $list->{'dir'} . '/archives/log.') {
                rename $list->{'dir'} . '/archives/log.',
                    $list->{'dir'} . '/archives/log.00';
            }
        }
302
303
    }

304
    if (lower_version($previous_version, '5.2a.1')) {
305

306
        ## Fill the robot_subscriber and robot_admin fields in DB
307
        $log->syslog('notice',
308
309
310
311
            'Updating the new robot_subscriber and robot_admin  Db fields...'
        );

        foreach my $r (keys %{$Conf::Conf{'robots'}}) {
sikeda's avatar
sikeda committed
312
313
            my $all_lists =
                Sympa::List::get_lists($r, 'skip_sync_admin' => 1);
314
            foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
315
                my $sdm = Sympa::DatabaseManager->instance;
316
317
                foreach my $table ('subscriber', 'admin') {
                    unless (
sikeda's avatar
sikeda committed
318
319
                        $sdm
                        and $sdm->do_query(
320
321
322
                            "UPDATE %s_table SET robot_%s=%s WHERE (list_%s=%s)",
                            $table,
                            $table,
sikeda's avatar
sikeda committed
323
                            $sdm->quote($r),
324
                            $table,
sikeda's avatar
sikeda committed
325
                            $sdm->quote($list->{'name'})
326
                        )
Luc Didry's avatar
Luc Didry committed
327
                    ) {
328
                        $log->syslog(
329
                            'err',
330
                            'Unable to fille the robot_admin and robot_subscriber fields in database for robot %s',
331
332
                            $r
                        );
333
                        Sympa::send_notify_to_listmaster('*',
sikeda's avatar
sikeda committed
334
                            'upgrade_failed', {'error' => $sdm->error});
335
336
337
338
339
340
                        return undef;
                    }
                }

                ## Force Sync_admin
                $list =
341
                    Sympa::List->new($list->{'name'}, $list->{'domain'},
342
343
344
345
346
                    {'force_sync_admin' => 1});
            }
        }

        ## Rename web archive directories using 'domain' instead of 'host'
347
        $log->syslog('notice',
348
349
350
351
352
            'Renaming web archive directories with the list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'arc_path');
        unless (opendir ARCDIR, $root_dir) {
353
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
354
355
356
357
358
359
360
361
362
363
364
365
            return undef;
        }

        foreach my $dir (sort readdir(ARCDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            my ($listname, $listdomain) = split /\@/, $dir;

            next unless ($listname && $listdomain);

366
            my $list = Sympa::List->new($listname, $listdomain);
367
            unless (defined $list) {
368
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
369
370
371
372
373
374
375
376
377
378
                next;
            }

            if ($listdomain ne $list->{'domain'}) {
                my $old_path =
                    $root_dir . '/' . $listname . '@' . $listdomain;
                my $new_path =
                    $root_dir . '/' . $listname . '@' . $list->{'domain'};

                if (-d $new_path) {
379
                    $log->syslog(
380
                        'err',
381
                        'Could not rename %s to %s; directory already exists',
382
383
384
385
386
387
                        $old_path,
                        $new_path
                    );
                    next;
                } else {
                    unless (rename $old_path, $new_path) {
388
                        $log->syslog('err', 'Failed to rename %s to %s: %m',
389
                            $old_path, $new_path);
390
391
                        next;
                    }
392
                    $log->syslog('notice', "Renamed %s to %s",
393
394
395
396
397
398
                        $old_path, $new_path);
                }
            }
        }
        close ARCDIR;

399
400
401
    }

    ## DB fields of enum type have been changed to int
402
    if (lower_version($previous_version, '5.2a.1')) {
403
        if ($Conf::Conf{'db_type'} eq 'mysql') {
404
405
406
407
408
409
410
411
412
413
414
            my %check = (
                'subscribed_subscriber' => 'subscriber_table',
                'included_subscriber'   => 'subscriber_table',
                'subscribed_admin'      => 'admin_table',
                'included_admin'        => 'admin_table'
            );

            foreach my $field (keys %check) {
                my $statement;
                my $sth;

sikeda's avatar
sikeda committed
415
416
417
418
419
420
421
                my $sdm = Sympa::DatabaseManager->instance;
                unless (
                    $sdm
                    and $sth = $sdm->do_query(
                        q{SELECT max(%s) FROM %s},
                        $field, $check{$field}
                    )
Luc Didry's avatar
Luc Didry committed
422
                ) {
423
                    $log->syslog('err', 'Unable to prepare SQL statement');
424
425
426
427
428
429
430
431
432
433
                    return undef;
                }

                my $max = $sth->fetchrow();
                $sth->finish();

                ## '0' has been mapped to 1 and '1' to 2
                ## Restore correct field value
                if ($max > 1) {
                    ## 1 to 0
434
                    $log->syslog('notice',
435
                        'Fixing DB field %s; turning 1 to 0...', $field);
436
437
                    my $rows;
                    $sth =
sikeda's avatar
sikeda committed
438
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
439
440
                        $check{$field}, $field, 0, $field, 1);
                    unless ($sth) {
441
442
                        $log->syslog('err',
                            'Unable to execute SQL statement');
443
444
445
                        return undef;
                    }
                    $rows = $sth->rows;
446
                    $log->syslog('notice', 'Updated %d rows', $rows);
447
448

                    ## 2 to 1
449
                    $log->syslog('notice',
450
                        'Fixing DB field %s; turning 2 to 1...', $field);
451
452

                    $sth =
sikeda's avatar
sikeda committed
453
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
454
455
                        $check{$field}, $field, 1, $field, 2);
                    unless ($sth) {
456
457
                        $log->syslog('err',
                            'Unable to execute SQL statement');
458
459
460
                        return undef;
                    }
                    $rows = $sth->rows;
461
                    $log->syslog('notice', 'Updated %d rows', $rows);
462
463
464
465
                }

                ## Set 'subscribed' data field to '1' is none of 'subscribed'
                ## and 'included' is set
466
                $log->syslog('notice',
467
468
                    'Updating subscribed field of the subscriber table...');
                my $rows;
sikeda's avatar
sikeda committed
469
                $sth = $sdm->do_query(
470
                    q{UPDATE subscriber_table
471
472
473
474
475
		      SET subscribed_subscriber = 1
		      WHERE (included_subscriber IS NULL OR
			     included_subscriber <> 1) AND
			    (subscribed_subscriber IS NULL OR
			     subscribed_subscriber <> 1)}
476
477
                );
                unless ($sth) {
478
                    $log->syslog('err', 'Unable to execute SQL statement');
479
480
481
                    return undef;
                }
                $rows = $sth->rows;
482
                $log->syslog('notice', '%d rows have been updated', $rows);
483
484
            }
        }
485
486
487
    }

    ## Rename bounce sub-directories
488
    if (lower_version($previous_version, '5.2a.1')) {
489

490
        $log->syslog('notice',
491
492
493
494
495
            'Renaming bounce sub-directories adding list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'bounce_path');
        unless (opendir BOUNCEDIR, $root_dir) {
496
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
497
498
499
500
501
502
503
504
505
506
507
508
509
            return undef;
        }

        foreach my $dir (sort readdir(BOUNCEDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            ## Directory already include the list domain
            next
                if ($dir =~ /\@/);

            my $listname = $dir;
510
            my $list     = Sympa::List->new($listname);
511
            unless (defined $list) {
512
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
513
514
515
516
517
518
519
520
                next;
            }

            my $old_path = $root_dir . '/' . $listname;
            my $new_path =
                $root_dir . '/' . $listname . '@' . $list->{'domain'};

            if (-d $new_path) {
521
                $log->syslog('err',
522
                    'Could not rename %s to %s; directory already exists',
523
524
525
526
                    $old_path, $new_path);
                next;
            } else {
                unless (rename $old_path, $new_path) {
527
                    $log->syslog('err', 'Failed to rename %s to %s: %m',
528
                        $old_path, $new_path);
529
530
                    next;
                }
531
                $log->syslog('notice', "Renamed %s to %s",
532
533
534
535
                    $old_path, $new_path);
            }
        }
        close BOUNCEDIR;
536
537
    }

538
    # Update lists config using 'include_sympa_list'
539
    if (lower_version($previous_version, '5.2a.1')) {
540
        $log->syslog('notice',
541
            'Update lists config using include_sympa_list parameter...');
542

543
        my $all_lists = Sympa::List::get_lists('*');
544
        foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
545
546
547
548
549
            my @include_lists =
                @{$list->{'admin'}{'include_sympa_list'} || []};
            my $changed = 0;
            foreach my $incl (@include_lists) {
                # Search for the list if robot is not specified.
550
551
                my $incl_list =
                    Sympa::List->new($incl->{listname}, $list->{'domain'});
sikeda's avatar
sikeda committed
552
553
554
555
556
557
558
559

                if (    $incl_list
                    and $incl_list->{'domain'} ne $list->{'domain'}) {
                    $log->syslog('notice',
                        'Update config file of list %s, including list %s',
                        $list->get_id, $incl_list->get_id);
                    $incl->{listname} = $incl_list->get_id;
                    $changed = 1;
560
                }
sikeda's avatar
sikeda committed
561
562
563
564
565
            }
            if ($changed) {
                $list->{'admin'}{'include_sympa_list'} = [@include_lists];
                $list->save_config(Sympa::get_address($list, 'listmaster'));
            }
566
        }
567
568
569
    }

    ## New mhonarc ressource file with utf-8 recoding
570
    if (lower_version($previous_version, '5.3a.6')) {
571

572
        $log->syslog('notice',
573
574
575
576
577
578
579
580
581
582
583
584
            'Looking for customized mhonarc-ressources.tt2 files...');
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            my $etc_dir = $Conf::Conf{'etc'};

            if ($vr ne $Conf::Conf{'domain'}) {
                $etc_dir .= '/' . $vr;
            }

            if (-f $etc_dir . '/mhonarc-ressources.tt2') {
                my $new_filename =
                    $etc_dir . '/mhonarc-ressources.tt2' . '.' . time;
                rename $etc_dir . '/mhonarc-ressources.tt2', $new_filename;
585
                $log->syslog(
586
587
588
589
590
                    'notice',
                    "Custom %s file has been backed up as %s",
                    $etc_dir . '/mhonarc-ressources.tt2',
                    $new_filename
                );
591
                Sympa::send_notify_to_listmaster('*', 'file_removed',
592
593
594
595
                    [$etc_dir . '/mhonarc-ressources.tt2', $new_filename]);
            }
        }

596
        $log->syslog('notice', 'Rebuilding web archives...');
597
        my $all_lists = Sympa::List::get_lists('*');
598
599
        foreach my $list (@$all_lists) {
            # FIXME: next line always success
600
601
            next
                unless defined $list->{'admin'}{'web_archive'}
602
                or defined $list->{'admin'}{'archive'};
603

604
605
606
607
608
609
610
611
612
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
613
614
615
                next;
            }
        }
616
617
618
619

    }

    ## Changed shared documents name encoding
620
621
    ## They are Q-encoded therefore easier to store on any filesystem with any
    ## encoding
622
    if (lower_version($previous_version, '5.3a.8')) {
623
        $log->syslog('notice', 'Q-Encoding web documents filenames...');
624
625
626
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists';
627
    }
628

629
630
    ## We now support UTF-8 only for custom templates, config files, headers
    ## and footers, info files
631
    ## + web_tt2, scenari, create_list_templatee, families
632
    if (lower_version($previous_version, '5.3b.3')) {
633
        $log->syslog('notice', 'Encoding all custom files to UTF-8...');
634
635
636
637
638
639
640
641

        my (@directories, @files);

        ## Site level
        foreach my $type (
            'mail_tt2', 'web_tt2',
            'scenari',  'create_list_templates',
            'families'
Luc Didry's avatar
Luc Didry committed
642
        ) {
643
644
645
646
647
648
649
650
651
652
653
            if (-d $Conf::Conf{'etc'} . '/' . $type) {
                push @directories,
                    [$Conf::Conf{'etc'} . '/' . $type, $Conf::Conf{'lang'}];
            }
        }

        foreach my $f (
            Conf::get_sympa_conf(),
            Conf::get_wwsympa_conf(),
            $Conf::Conf{'etc'} . '/' . 'topics.conf',
            $Conf::Conf{'etc'} . '/' . 'auth.conf'
Luc Didry's avatar
Luc Didry committed
654
        ) {
655
656
657
658
659
660
661
662
663
664
665
            if (-f $f) {
                push @files, [$f, $Conf::Conf{'lang'}];
            }
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            foreach my $type (
                'mail_tt2', 'web_tt2',
                'scenari',  'create_list_templates',
                'families'
Luc Didry's avatar
Luc Didry committed
666
            ) {
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
                if (-d $Conf::Conf{'etc'} . '/' . $vr . '/' . $type) {
                    push @directories,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $type,
                        Conf::get_robot_conf($vr, 'lang')
                        ];
                }
            }

            foreach my $f ('robot.conf', 'topics.conf', 'auth.conf') {
                if (-f $Conf::Conf{'etc'} . '/' . $vr . '/' . $f) {
                    push @files,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $f,
                        $Conf::Conf{'lang'}
                        ];
                }
            }
        }

        ## Search in Lists
688
        my $all_lists = Sympa::List::get_lists('*');
689
690
691
692
693
        foreach my $list (@$all_lists) {
            foreach my $f (
                'config',   'info',
                'homepage', 'message.header',
                'message.footer'
Luc Didry's avatar
Luc Didry committed
694
            ) {
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
                if (-f $list->{'dir'} . '/' . $f) {
                    push @files,
                        [$list->{'dir'} . '/' . $f, $list->{'admin'}{'lang'}];
                }
            }

            foreach my $type ('mail_tt2', 'web_tt2', 'scenari') {
                my $directory = $list->{'dir'} . '/' . $type;
                if (-d $directory) {
                    push @directories, [$directory, $list->{'admin'}{'lang'}];
                }
            }
        }

        ## Search language directories
        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }

            if ($d =~ /(mail_tt2|web_tt2)$/) {
                foreach
                    my $subdir (grep(/^[a-z]{2}(_[A-Z]{2})?$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories, ["$d/$subdir", $subdir];
                    }
                }
                closedir DIR;

            } elsif ($d =~ /(create_list_templates|families)$/) {
                foreach my $subdir (grep(/^\w+$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories,
                            ["$d/$subdir", $Conf::Conf{'lang'}];
                    }
                }
                closedir DIR;
            }
        }

        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }
            foreach my $file (readdir DIR) {
                next
                    unless (
                    (   $d =~
                        /mail_tt2|web_tt2|create_list_templates|families/
                        && $file =~ /\.tt2$/
                    )
                    || ($d =~ /scenari$/ && $file =~ /\w+\.\w+$/)
                    );
                push @files, [$d . '/' . $file, $lang];
            }
            closedir DIR;
        }

        ## Do the encoding modifications
        ## Previous versions of files are backed up with the date extension
        my $total = to_utf8(\@files);
758
        $log->syslog('notice', '%d files have been modified', $total);
759
760
    }

761
762
    ## giving up subscribers flat files ; moving subscribers to the DB
    ## Also giving up old 'database' mode
763
    if (lower_version($previous_version, '5.4a.1')) {
764

765
        $log->syslog('notice',
766
767
768
            'Looking for lists with user_data_source parameter set to file or database...'
        );

769
        my $all_lists = Sympa::List::get_lists('*');
770
771
772
773
        foreach my $list (@$all_lists) {

            if ($list->{'admin'}{'user_data_source'} eq 'file') {

774
                $log->syslog(
775
                    'notice',
776
                    'List %s; changing user_data_source from file to include2...',
777
778
779
                    $list->{'name'}
                );

780
781
782
783
                # Load <list dir>/subscribers to the DB
                if (-e $list->{'dir'} . '/subscribers'
                    and rename $list->{'dir'} . '/subscribers',
                    $list->{'dir'} . '/member.dump'
Luc Didry's avatar
Luc Didry committed
784
                ) {
785
                    $list->restore_users('member');
786

787
788
789
                    my $total = $list->{'add_outcome'}{'added_members'};
                    if (defined $list->{'add_outcome'}{'errors'}) {
                        $log->syslog('err', 'Failed to add users: %s',
Luc Didry's avatar
Luc Didry committed
790
791
                            $list->{'add_outcome'}{'errors'}{'error_message'}
                        );
792
793
794
795
                    }
                    $log->syslog('notice',
                        '%d subscribers have been loaded into the database',
                        $total);
796
797
                }

798
                $list->{'admin'}{'user_data_source'} = 'include2';
799
800

                unless ($list->save_config('automatic')) {
801
                    $log->syslog('err',
802
                        'Failed to save config file for list %s', $list);
803
804
805
                }
            } elsif ($list->{'admin'}{'user_data_source'} eq 'database') {

806
                $log->syslog(
807
                    'notice',
808
                    'List %s; changing user_data_source from database to include2...',
809
810
811
812
                    $list->{'name'}
                );

                unless ($list->update_list_member('*', {'subscribed' => 1})) {
813
                    $log->syslog('err',
814
815
816
817
818
819
                        'Failed to update subscribed DB field');
                }

                $list->{'admin'}{'user_data_source'} = 'include2';

                unless ($list->save_config('automatic')) {
820
                    $log->syslog('err',
821
822
823
824
825
                        'Failed to save config file for list %s',
                        $list->{'name'});
                }
            }
        }
826
    }
827

828
    if (lower_version($previous_version, '5.5a.1')) {
sikeda's avatar
sikeda committed
829
        # Remove OTHER/ subdirectories in bounces
830
        $log->syslog('notice', "Removing obsolete OTHER/ bounce directories");
sikeda's avatar
sikeda committed
831
832
        if (opendir my $dh, $Conf::Conf{'bounce_path'}) {
            foreach my $subdir (sort grep (!/^\.+$/, readdir $dh)) {
833
                my $other_dir =
sikeda's avatar
sikeda committed
834
                    $Conf::Conf{'bounce_path'} . '/' . $subdir . '/OTHER';
835
                if (-d $other_dir) {
836
                    Sympa::Tools::File::remove_dir($other_dir);
837
838
                    $log->syslog('notice', 'Directory %s removed',
                        $other_dir);
839
840
                }
            }
sikeda's avatar
sikeda committed
841
            closedir $dh;
842
        } else {
843
            $log->syslog(
844
845
                'err',
                'Failed to open directory %s: %m',
sikeda's avatar
sikeda committed
846
                $Conf::Conf{'bounce_path'}
847
            );
848
849
850
        }
    }

851
    if (lower_version($previous_version, '6.1b.5')) {
852
853
854
855
856
        ## Encoding of shared documents was not consistent with recent
        ## versions of MIME::Encode
        ## MIME::EncWords::encode_mimewords() used to encode characters -!*+/
        ## Now these characters are preserved, according to RFC 2047 section 5
        ## We change encoding of shared documents according to new algorithm
857
        $log->syslog('notice',
858
            'Fixing Q-encoding of web document filenames...');
859
860
861
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists', '--fix_qencode';
862
    }
863
    if (lower_version($previous_version, '6.1.11')) {
864
        ## Exclusion table was not robot-enabled.
865
        $log->syslog('notice', 'Fixing robot column of exclusion table');
866
        my $sth;
sikeda's avatar
sikeda committed
867
868
869
        my $sdm = Sympa::DatabaseManager->instance;
        unless ($sdm
            and $sth = $sdm->do_query("SELECT * FROM exclusion_table")) {
870
            $log->syslog('err',
871
                'Unable to gather information from the exclusions table');
872
        }
873
        my @robots = Sympa::List::get_robots();
874
875
876
877
878
879
880
881
        while (my $data = $sth->fetchrow_hashref) {
            next
                if (defined $data->{'robot_exclusion'}
                && $data->{'robot_exclusion'} ne '');
            ## Guessing right robot for each exclusion.
            my $valid_robot = '';
            my @valid_robot_candidates;
            foreach my $robot (@robots) {
sikeda's avatar
sikeda committed
882
883
                if (my $list =
                    Sympa::List->new($data->{'list_exclusion'}, $robot)) {
884
885
886
887
888
889
890
891
892
                    if ($list->is_list_member($data->{'user_exclusion'})) {
                        push @valid_robot_candidates, $robot;
                    }
                }
            }
            if ($#valid_robot_candidates == 0) {
                $valid_robot = $valid_robot_candidates[0];
                my $sth;
                unless (
sikeda's avatar
sikeda committed
893
                    $sth = $sdm->do_query(
894
                        "UPDATE exclusion_table SET robot_exclusion = %s WHERE list_exclusion=%s AND user_exclusion=%s",
sikeda's avatar
sikeda committed
895
896
897
                        $sdm->quote($valid_robot),
                        $sdm->quote($data->{'list_exclusion'}),
                        $sdm->quote($data->{'user_exclusion'})
898
                    )
Luc Didry's avatar
Luc Didry committed
899
                ) {
900
                    $log->syslog(
901
                        'err',
902
                        'Unable to update entry (%s, %s) in exclusions table (trying to add robot %s)',
903
904
905
906
907
908
                        $data->{'list_exclusion'},
                        $data->{'user_exclusion'},
                        $valid_robot
                    );
                }
            } else {
909
                $log->syslog(
910
                    'err',
911
                    'Exclusion robot could not be guessed for user "%s" in list "%s". Either this user is no longer subscribed to the list or the list appears in more than one robot (or the query to the database failed). Here is the list of robots in which this list name appears: "%s"',
912
913
914
915
916
917
918
                    $data->{'user_exclusion'},
                    $data->{'list_exclusion'},
                    @valid_robot_candidates
                );
            }
        }
        ## Caching all lists config subset to database
919
        $log->syslog('notice', 'Caching all list config to database...');
920
921
        Sympa::List::_flush_list_db();
        my $all_lists = Sympa::List::get_lists('*', 'reload_config' => 1);
922
923
924
        foreach my $list (@$all_lists) {
            $list->_update_list_db;
        }
925
        $log->syslog('notice', '...done');
926
    }
927

928
    ## We have obsoleted wwsympa.conf.  It would be migrated to sympa.conf.
929
    if (lower_version($previous_version, '6.2b.1')) {
930
931
        $log->syslog('notice', 'Migrating wwsympa.conf...');

932
933
934
935
936
        my $sympa_conf   = Conf::get_sympa_conf();
        my $wwsympa_conf = Conf::get_wwsympa_conf();
        my $fh;
        my %migrated = ();
        my @newconf  = ();
937
        my ($date, $human_date);
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974

        ## Some sympa.conf parameters were overridden by wwsympa.conf.
        ## Others prefer sympa.conf.
        my %wwsconf_override = (
            'arc_path'                   => 'yes',
            'archive_default_index'      => 'yes',
            'bounce_path'                => 'yes',
            'cookie_domain'              => 'NO',
            'cookie_expire'              => 'yes',
            'cookie_refresh'             => 'NO',
            'custom_archiver'            => 'yes',
            'default_home'               => 'NO',
            'export_topics'              => 'yes',
            'html_editor_file'           => 'NO',    # 6.2a
            'html_editor_init'           => 'NO',
            'ldap_force_canonical_email' => 'NO',
            'log_facility'               => 'yes',
            'mhonarc'                    => 'yes',
            'password_case'              => 'NO',
            'review_page_size'           => 'yes',
            'title'                      => 'NO',
            'use_html_editor'            => 'NO',
            'viewlogs_page_size'         => 'yes',
            'wws_path'                   => undef,
        );
        ## Old params
        my %old_param = (
            'alias_manager' => 'No more used, using '
                . $Conf::Conf{'alias_manager'},
            'wws_path' => 'No more used',
            'icons_url' =>
                'No more used. Using static_content/icons instead.',
            'robots' =>
                'Not used anymore. Robots are fully described in their respective robot.conf file.',
            'task_manager_pidfile' => 'No more used',
            'archived_pidfile'     => 'No more used',
            'bounced_pidfile'      => 'No more used',
975
            'use_fast_cgi'         => 'No longer used',   # 6.2.25b deprecated
IKEDA Soji's avatar
IKEDA Soji committed
976
            'htmlarea_url'         => 'No longer used',   # 6.2.36 deprecated
977
978
979
980
        );

        ## Set language of new file content
        $language->push_lang($Conf::Conf{'lang'});
Sympa authors's avatar
Sympa authors committed
981
        $date       = time;
982
983
        $human_date = $language->gettext_strftime('%d %b %Y at %H:%M:%S',
            localtime $date);
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999

        if (-r $wwsympa_conf) {
            ## load only sympa.conf
            my $conf = (
                Conf::_load_config_file_to_hash(
                    {'path_to_config_file' => $sympa_conf}
                    )
                    || {}
            )->{'config'};
            # not yet implemented.
            #my $conf = Conf::load_robot_conf(
            #    {'robot' => '*', 'no_db' => 1, 'return_result' => 1}
            #);

            my %infile = ();
            ## load defaults
1000
            foreach my $p (@Sympa::ConfDef::params) {
For faster browsing, not all history is shown. View entire blame