Upgrade.pm 80.6 KB
Newer Older
1
2
3
# -*- indent-tabs-mode: nil; -*-
# vim:ft=perl:et:sw=4
# $Id$
4
5

# Sympa - SYsteme de Multi-Postage Automatique
6
7
8
9
#
# Copyright (c) 1997, 1998, 1999 Institut Pasteur & Christophe Wolfhugel
# Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
# 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
10
# Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
11
12
# Copyright 2017, 2018 The Sympa Community. See the AUTHORS.md file at the
# top-level directory of this distribution and at
13
# <https://github.com/sympa-community/sympa.git>.
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
26
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
27

28
package Sympa::Upgrade;
29
30

use strict;
31
use warnings;
32
use Encode qw();
33
use English qw(-no_match_vars);
34
use MIME::Base64 qw();
35
use Time::Local qw();
36

37
use Sympa;
38
use Conf;
39
use Sympa::ConfDef;
40
use Sympa::Constants;
sikeda's avatar
sikeda committed
41
use Sympa::DatabaseManager;
42
use Sympa::Language;
43
use Sympa::List;
44
use Sympa::LockedFile;
45
use Sympa::Log;
46
use Sympa::Message;
47
use Sympa::Request;
IKEDA Soji's avatar
IKEDA Soji committed
48
use Sympa::Scenario;
49
use Sympa::Spool;
50
use Sympa::Spool::Archive;
51
use Sympa::Spool::Auth;
52
use Sympa::Spool::Digest;
53
use Sympa::Tools::Data;
54
55
use Sympa::Tools::File;
use Sympa::Tools::Text;
56

57
my $language = Sympa::Language->instance;
58
my $log      = Sympa::Log->instance;
59

60
61
## Return the previous Sympa version, ie the one listed in
## data_structure.version
62
sub get_previous_version {
63
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
64
    my $previous_version;
65

66
    if (-f $version_file) {
67
        unless (open VFILE, $version_file) {
68
            $log->syslog('err', 'Unable to open %s: %m', $version_file);
69
70
71
72
73
74
75
76
77
78
79
80
            return undef;
        }
        while (<VFILE>) {
            next if /^\s*$/;
            next if /^\s*\#/;
            chomp;
            $previous_version = $_;
            last;
        }
        close VFILE;

        return $previous_version;
81
    }
82

83
84
85
86
    return undef;
}

sub update_version {
87
    my $version_file = "$Conf::Conf{'etc'}/data_structure.version";
88
89
90

    ## Saving current version if required
    unless (open VFILE, ">$version_file") {
91
        $log->syslog(
92
            'err',
93
            'Unable to write %s; sympa.pl needs write access on %s directory: %m',
94
            $version_file,
95
            $Conf::Conf{'etc'}
96
97
        );
        return undef;
98
    }
99
100
    printf VFILE
        "# This file is automatically created by sympa.pl after installation\n# Unless you know what you are doing, you should not modify it\n";
101
    printf VFILE "%s\n", Sympa::Constants::VERSION;
102
    close VFILE;
103

104
105
106
107
108
    return 1;
}

## Upgrade data structure from one version to another
sub upgrade {
109
    $log->syslog('debug3', '(%s, %s)', @_);
110
111
    my ($previous_version, $new_version) = @_;

112
    if (lower_version($new_version, $previous_version)) {
113
        $log->syslog('notice',
114
115
116
            'Installing  older version of Sympa ; no upgrade operation is required'
        );
        return 1;
117
118
    }

119
    ## Check database connectivity and probe database
sikeda's avatar
sikeda committed
120
    unless (Sympa::DatabaseManager::probe_db()) {
121
        $log->syslog(
122
            'err',
123
124
125
126
127
128
            'Database %s defined in sympa.conf has not the right structure or is unreachable. verify db_xxx parameters in sympa.conf',
            $Conf::Conf{'db_name'}
        );
        return undef;
    }

129
130
131
132
133
    # As of 6.2.33b.1, owners/moderators are no longer stored in config file.
    # - Write out initial permanent owners/editors in <role>.dump files.
    # - And, if list is not closed, import owners/moderators from those files
    #   into database.
    if (lower_version($previous_version, '6.2.33b.1')) {
IKEDA Soji's avatar
IKEDA Soji committed
134
135
136
        $log->syslog('notice',
            'Restoring users of ALL lists...it may take a while...');

137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        my $all_lists = Sympa::List::get_lists('*', skip_sync_admin => 1);
        foreach my $list (@{$all_lists || []}) {
            next unless $list;
            my $dir = $list->{'dir'};

            my $fh;
            next unless open $fh, '<', $dir . '/config';
            my $config = do { local $RS; <$fh> };
            close $fh;

            $config =~ s/(\A|\n)[\t ]+(?=\n)/$1/g;    # normalize empty lines
            open my $ifh, '<', \$config;              # open "in memory" file
            my @config = do { local $RS = ''; <$ifh> };
            close $ifh;
            foreach my $role (qw(owner editor)) {
                my $file = $dir . '/' . $role . '.dump';
                if (!-e $file and open my $ofh, '>', $file) {
                    my $admins = join '', grep {/\A\s*$role\b/} @config;
                    print $ofh $admins;
                    close $ofh;
                }

                next
                    if $list->{'admin'}{'status'} eq 'closed'
                    or $list->{'admin'}{'status'} eq 'family_closed';
                $list->restore_users($role);
            }
        }
    }

IKEDA Soji's avatar
IKEDA Soji committed
167
168
    # Always update config.bin files while upgrading.
    # This is especially useful for character encoding reasons.
169
    $log->syslog('notice',
170
        'Rebuilding config.bin files for ALL lists...it may take a while...');
IKEDA Soji's avatar
IKEDA Soji committed
171
172
    my $all_lists =
        Sympa::List::get_lists('*', reload_config => 1, skip_sync_admin => 1);
IKEDA Soji's avatar
IKEDA Soji committed
173
174
175
176
177
    # Recreate admin_table entries.
    $log->syslog('notice',
        'Rebuilding the admin_table...it may take a while...');
    foreach my $list (@{$all_lists || []}) {    # See GH #71
        $list->sync_include_admin;
178
    }
179

180
    ## Migration to tt2
181
    if (lower_version($previous_version, '4.2b')) {
182

183
        $log->syslog('notice', 'Migrating templates to TT2 format...');
184
185
186

        my $tpl_script = Sympa::Constants::SCRIPTDIR . '/tpl2tt2.pl';
        unless (open EXEC, "$tpl_script|") {
187
            $log->syslog('err', 'Unable to run %s', $tpl_script);
188
189
190
191
            return undef;
        }
        close EXEC;

192
        $log->syslog('notice', 'Rebuilding web archives...');
193
        my $all_lists = Sympa::List::get_lists('*');
194
195
        foreach my $list (@$all_lists) {
            # FIXME: line below will always success
196
197
            next
                unless defined $list->{'admin'}{'web_archive'}
198
                or defined $list->{'admin'}{'archive'};
199

200
201
202
203
204
205
206
207
208
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
209
210
211
                next;
            }
        }
212
    }
213

214
    ## Initializing the new admin_table
215
    if (lower_version($previous_version, '4.2b.4')) {
216
        $log->syslog('notice', 'Initializing the new admin_table...');
217
        my $all_lists = Sympa::List::get_lists('*');
218
219
220
        foreach my $list (@$all_lists) {
            $list->sync_include_admin();
        }
221
222
223
    }

    ## Move old-style web templates out of the include_path
224
    if (lower_version($previous_version, '5.0.1')) {
225
        $log->syslog('notice',
226
227
            'Old web templates HTML structure is not compliant with latest ones.'
        );
228
        $log->syslog('notice',
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
            'Moving old-style web templates out of the include_path...');

        my @directories;

        if (-d "$Conf::Conf{'etc'}/web_tt2") {
            push @directories, "$Conf::Conf{'etc'}/web_tt2";
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {

            if (-d "$Conf::Conf{'etc'}/$vr/web_tt2") {
                push @directories, "$Conf::Conf{'etc'}/$vr/web_tt2";
            }
        }

        ## Search in V. Robot Lists
246
        my $all_lists = Sympa::List::get_lists('*');
247
248
249
250
251
252
253
254
255
256
        foreach my $list (@$all_lists) {
            if (-d "$list->{'dir'}/web_tt2") {
                push @directories, "$list->{'dir'}/web_tt2";
            }
        }

        my @templates;

        foreach my $d (@directories) {
            unless (opendir DIR, $d) {
257
258
                printf STDERR "Error: Cannot read %s directory: %s", $d,
                    $ERRNO;
259
260
261
262
263
264
265
266
267
268
269
270
271
                next;
            }

            foreach my $tt2 (sort grep(/\.tt2$/, readdir DIR)) {
                push @templates, "$d/$tt2";
            }

            closedir DIR;
        }

        foreach my $tpl (@templates) {
            unless (rename $tpl, "$tpl.oldtemplate") {
                printf STDERR
272
273
                    "Error : failed to rename %s to %s.oldtemplate: %s\n",
                    $tpl, $tpl, $ERRNO;
274
275
                next;
            }
276

277
            $log->syslog('notice', 'File %s renamed %s',
278
279
280
                $tpl, "$tpl.oldtemplate");
        }
    }
281
282

    ## Clean buggy list config files
283
    if (lower_version($previous_version, '5.1b')) {
284
        $log->syslog('notice', 'Cleaning buggy list config files...');
285
        my $all_lists = Sympa::List::get_lists('*');
286
287
288
        foreach my $list (@$all_lists) {
            $list->save_config('listmaster@' . $list->{'domain'});
        }
289
290
291
    }

    ## Fix a bug in Sympa 5.1
292
    if (lower_version($previous_version, '5.1.2')) {
293
        $log->syslog('notice', 'Rename archives/log. files...');
294
        my $all_lists = Sympa::List::get_lists('*');
295
296
297
298
299
300
        foreach my $list (@$all_lists) {
            if (-f $list->{'dir'} . '/archives/log.') {
                rename $list->{'dir'} . '/archives/log.',
                    $list->{'dir'} . '/archives/log.00';
            }
        }
301
302
    }

303
    if (lower_version($previous_version, '5.2a.1')) {
304

305
        ## Fill the robot_subscriber and robot_admin fields in DB
306
        $log->syslog('notice',
307
308
309
310
            'Updating the new robot_subscriber and robot_admin  Db fields...'
        );

        foreach my $r (keys %{$Conf::Conf{'robots'}}) {
sikeda's avatar
sikeda committed
311
312
            my $all_lists =
                Sympa::List::get_lists($r, 'skip_sync_admin' => 1);
313
            foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
314
                my $sdm = Sympa::DatabaseManager->instance;
315
316
                foreach my $table ('subscriber', 'admin') {
                    unless (
sikeda's avatar
sikeda committed
317
318
                        $sdm
                        and $sdm->do_query(
319
320
321
                            "UPDATE %s_table SET robot_%s=%s WHERE (list_%s=%s)",
                            $table,
                            $table,
sikeda's avatar
sikeda committed
322
                            $sdm->quote($r),
323
                            $table,
sikeda's avatar
sikeda committed
324
                            $sdm->quote($list->{'name'})
325
                        )
Luc Didry's avatar
Luc Didry committed
326
                    ) {
327
                        $log->syslog(
328
                            'err',
329
                            'Unable to fille the robot_admin and robot_subscriber fields in database for robot %s',
330
331
                            $r
                        );
332
                        Sympa::send_notify_to_listmaster('*',
sikeda's avatar
sikeda committed
333
                            'upgrade_failed', {'error' => $sdm->error});
334
335
336
337
338
339
                        return undef;
                    }
                }

                ## Force Sync_admin
                $list =
340
                    Sympa::List->new($list->{'name'}, $list->{'domain'},
341
342
343
344
345
                    {'force_sync_admin' => 1});
            }
        }

        ## Rename web archive directories using 'domain' instead of 'host'
346
        $log->syslog('notice',
347
348
349
350
351
            'Renaming web archive directories with the list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'arc_path');
        unless (opendir ARCDIR, $root_dir) {
352
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
353
354
355
356
357
358
359
360
361
362
363
364
            return undef;
        }

        foreach my $dir (sort readdir(ARCDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            my ($listname, $listdomain) = split /\@/, $dir;

            next unless ($listname && $listdomain);

365
            my $list = Sympa::List->new($listname, $listdomain);
366
            unless (defined $list) {
367
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
368
369
370
371
372
373
374
375
376
377
                next;
            }

            if ($listdomain ne $list->{'domain'}) {
                my $old_path =
                    $root_dir . '/' . $listname . '@' . $listdomain;
                my $new_path =
                    $root_dir . '/' . $listname . '@' . $list->{'domain'};

                if (-d $new_path) {
378
                    $log->syslog(
379
                        'err',
380
                        'Could not rename %s to %s; directory already exists',
381
382
383
384
385
386
                        $old_path,
                        $new_path
                    );
                    next;
                } else {
                    unless (rename $old_path, $new_path) {
387
                        $log->syslog('err', 'Failed to rename %s to %s: %m',
388
                            $old_path, $new_path);
389
390
                        next;
                    }
391
                    $log->syslog('notice', "Renamed %s to %s",
392
393
394
395
396
397
                        $old_path, $new_path);
                }
            }
        }
        close ARCDIR;

398
399
400
    }

    ## DB fields of enum type have been changed to int
401
    if (lower_version($previous_version, '5.2a.1')) {
402
        if ($Conf::Conf{'db_type'} eq 'mysql') {
403
404
405
406
407
408
409
410
411
412
413
            my %check = (
                'subscribed_subscriber' => 'subscriber_table',
                'included_subscriber'   => 'subscriber_table',
                'subscribed_admin'      => 'admin_table',
                'included_admin'        => 'admin_table'
            );

            foreach my $field (keys %check) {
                my $statement;
                my $sth;

sikeda's avatar
sikeda committed
414
415
416
417
418
419
420
                my $sdm = Sympa::DatabaseManager->instance;
                unless (
                    $sdm
                    and $sth = $sdm->do_query(
                        q{SELECT max(%s) FROM %s},
                        $field, $check{$field}
                    )
Luc Didry's avatar
Luc Didry committed
421
                ) {
422
                    $log->syslog('err', 'Unable to prepare SQL statement');
423
424
425
426
427
428
429
430
431
432
                    return undef;
                }

                my $max = $sth->fetchrow();
                $sth->finish();

                ## '0' has been mapped to 1 and '1' to 2
                ## Restore correct field value
                if ($max > 1) {
                    ## 1 to 0
433
                    $log->syslog('notice',
434
                        'Fixing DB field %s; turning 1 to 0...', $field);
435
436
                    my $rows;
                    $sth =
sikeda's avatar
sikeda committed
437
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
438
439
                        $check{$field}, $field, 0, $field, 1);
                    unless ($sth) {
440
441
                        $log->syslog('err',
                            'Unable to execute SQL statement');
442
443
444
                        return undef;
                    }
                    $rows = $sth->rows;
445
                    $log->syslog('notice', 'Updated %d rows', $rows);
446
447

                    ## 2 to 1
448
                    $log->syslog('notice',
449
                        'Fixing DB field %s; turning 2 to 1...', $field);
450
451

                    $sth =
sikeda's avatar
sikeda committed
452
                        $sdm->do_query(q{UPDATE %s SET %s = %d WHERE %s = %d},
453
454
                        $check{$field}, $field, 1, $field, 2);
                    unless ($sth) {
455
456
                        $log->syslog('err',
                            'Unable to execute SQL statement');
457
458
459
                        return undef;
                    }
                    $rows = $sth->rows;
460
                    $log->syslog('notice', 'Updated %d rows', $rows);
461
462
463
464
                }

                ## Set 'subscribed' data field to '1' is none of 'subscribed'
                ## and 'included' is set
465
                $log->syslog('notice',
466
467
                    'Updating subscribed field of the subscriber table...');
                my $rows;
sikeda's avatar
sikeda committed
468
                $sth = $sdm->do_query(
469
                    q{UPDATE subscriber_table
470
471
472
473
474
		      SET subscribed_subscriber = 1
		      WHERE (included_subscriber IS NULL OR
			     included_subscriber <> 1) AND
			    (subscribed_subscriber IS NULL OR
			     subscribed_subscriber <> 1)}
475
476
                );
                unless ($sth) {
477
                    $log->syslog('err', 'Unable to execute SQL statement');
478
479
480
                    return undef;
                }
                $rows = $sth->rows;
481
                $log->syslog('notice', '%d rows have been updated', $rows);
482
483
            }
        }
484
485
486
    }

    ## Rename bounce sub-directories
487
    if (lower_version($previous_version, '5.2a.1')) {
488

489
        $log->syslog('notice',
490
491
492
493
494
            'Renaming bounce sub-directories adding list domain...');

        my $root_dir =
            Conf::get_robot_conf($Conf::Conf{'domain'}, 'bounce_path');
        unless (opendir BOUNCEDIR, $root_dir) {
495
            $log->syslog('err', 'Unable to open %s: %m', $root_dir);
496
497
498
499
500
501
502
503
504
505
506
507
508
            return undef;
        }

        foreach my $dir (sort readdir(BOUNCEDIR)) {
            ## Skip files and entries starting with '.'
            next
                if (($dir =~ /^\./o) || (!-d $root_dir . '/' . $dir));

            ## Directory already include the list domain
            next
                if ($dir =~ /\@/);

            my $listname = $dir;
509
            my $list     = Sympa::List->new($listname);
510
            unless (defined $list) {
511
                $log->syslog('notice', 'Skipping unknown list %s', $listname);
512
513
514
515
516
517
518
519
                next;
            }

            my $old_path = $root_dir . '/' . $listname;
            my $new_path =
                $root_dir . '/' . $listname . '@' . $list->{'domain'};

            if (-d $new_path) {
520
                $log->syslog('err',
521
                    'Could not rename %s to %s; directory already exists',
522
523
524
525
                    $old_path, $new_path);
                next;
            } else {
                unless (rename $old_path, $new_path) {
526
                    $log->syslog('err', 'Failed to rename %s to %s: %m',
527
                        $old_path, $new_path);
528
529
                    next;
                }
530
                $log->syslog('notice', "Renamed %s to %s",
531
532
533
534
                    $old_path, $new_path);
            }
        }
        close BOUNCEDIR;
535
536
    }

537
    # Update lists config using 'include_sympa_list'
538
    if (lower_version($previous_version, '5.2a.1')) {
539
        $log->syslog('notice',
540
            'Update lists config using include_sympa_list parameter...');
541

542
        my $all_lists = Sympa::List::get_lists('*');
543
        foreach my $list (@$all_lists) {
sikeda's avatar
sikeda committed
544
545
546
547
548
            my @include_lists =
                @{$list->{'admin'}{'include_sympa_list'} || []};
            my $changed = 0;
            foreach my $incl (@include_lists) {
                # Search for the list if robot is not specified.
549
550
                my $incl_list =
                    Sympa::List->new($incl->{listname}, $list->{'domain'});
sikeda's avatar
sikeda committed
551
552
553
554
555
556
557
558

                if (    $incl_list
                    and $incl_list->{'domain'} ne $list->{'domain'}) {
                    $log->syslog('notice',
                        'Update config file of list %s, including list %s',
                        $list->get_id, $incl_list->get_id);
                    $incl->{listname} = $incl_list->get_id;
                    $changed = 1;
559
                }
sikeda's avatar
sikeda committed
560
561
562
563
564
            }
            if ($changed) {
                $list->{'admin'}{'include_sympa_list'} = [@include_lists];
                $list->save_config(Sympa::get_address($list, 'listmaster'));
            }
565
        }
566
567
568
    }

    ## New mhonarc ressource file with utf-8 recoding
569
    if (lower_version($previous_version, '5.3a.6')) {
570

571
        $log->syslog('notice',
572
573
574
575
576
577
578
579
580
581
582
583
            'Looking for customized mhonarc-ressources.tt2 files...');
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            my $etc_dir = $Conf::Conf{'etc'};

            if ($vr ne $Conf::Conf{'domain'}) {
                $etc_dir .= '/' . $vr;
            }

            if (-f $etc_dir . '/mhonarc-ressources.tt2') {
                my $new_filename =
                    $etc_dir . '/mhonarc-ressources.tt2' . '.' . time;
                rename $etc_dir . '/mhonarc-ressources.tt2', $new_filename;
584
                $log->syslog(
585
586
587
588
589
                    'notice',
                    "Custom %s file has been backed up as %s",
                    $etc_dir . '/mhonarc-ressources.tt2',
                    $new_filename
                );
590
                Sympa::send_notify_to_listmaster('*', 'file_removed',
591
592
593
594
                    [$etc_dir . '/mhonarc-ressources.tt2', $new_filename]);
            }
        }

595
        $log->syslog('notice', 'Rebuilding web archives...');
596
        my $all_lists = Sympa::List::get_lists('*');
597
598
        foreach my $list (@$all_lists) {
            # FIXME: next line always success
599
600
            next
                unless defined $list->{'admin'}{'web_archive'}
601
                or defined $list->{'admin'}{'archive'};
602

603
604
605
606
607
608
609
610
611
            my $arc_message = Sympa::Message->new(
                sprintf("\nrebuildarc %s *\n\n", $list->{'name'}),
                context => $list->{'domain'},
                sender  => sprintf('listmaster@%s', $list->{'domain'}),
                date    => time
            );
            unless (Sympa::Spool::Archive->new->store($arc_message)) {
                $log->syslog('err', 'Cannot rebuild web archive of %s',
                    $list);
612
613
614
                next;
            }
        }
615
616
617
618

    }

    ## Changed shared documents name encoding
619
620
    ## They are Q-encoded therefore easier to store on any filesystem with any
    ## encoding
621
    if (lower_version($previous_version, '5.3a.8')) {
622
        $log->syslog('notice', 'Q-Encoding web documents filenames...');
623
624
625
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists';
626
    }
627

628
629
    ## We now support UTF-8 only for custom templates, config files, headers
    ## and footers, info files
630
    ## + web_tt2, scenari, create_list_templatee, families
631
    if (lower_version($previous_version, '5.3b.3')) {
632
        $log->syslog('notice', 'Encoding all custom files to UTF-8...');
633
634
635
636
637
638
639
640

        my (@directories, @files);

        ## Site level
        foreach my $type (
            'mail_tt2', 'web_tt2',
            'scenari',  'create_list_templates',
            'families'
Luc Didry's avatar
Luc Didry committed
641
        ) {
642
643
644
645
646
647
648
649
650
651
652
            if (-d $Conf::Conf{'etc'} . '/' . $type) {
                push @directories,
                    [$Conf::Conf{'etc'} . '/' . $type, $Conf::Conf{'lang'}];
            }
        }

        foreach my $f (
            Conf::get_sympa_conf(),
            Conf::get_wwsympa_conf(),
            $Conf::Conf{'etc'} . '/' . 'topics.conf',
            $Conf::Conf{'etc'} . '/' . 'auth.conf'
Luc Didry's avatar
Luc Didry committed
653
        ) {
654
655
656
657
658
659
660
661
662
663
664
            if (-f $f) {
                push @files, [$f, $Conf::Conf{'lang'}];
            }
        }

        ## Go through Virtual Robots
        foreach my $vr (keys %{$Conf::Conf{'robots'}}) {
            foreach my $type (
                'mail_tt2', 'web_tt2',
                'scenari',  'create_list_templates',
                'families'
Luc Didry's avatar
Luc Didry committed
665
            ) {
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
                if (-d $Conf::Conf{'etc'} . '/' . $vr . '/' . $type) {
                    push @directories,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $type,
                        Conf::get_robot_conf($vr, 'lang')
                        ];
                }
            }

            foreach my $f ('robot.conf', 'topics.conf', 'auth.conf') {
                if (-f $Conf::Conf{'etc'} . '/' . $vr . '/' . $f) {
                    push @files,
                        [
                        $Conf::Conf{'etc'} . '/' . $vr . '/' . $f,
                        $Conf::Conf{'lang'}
                        ];
                }
            }
        }

        ## Search in Lists
687
        my $all_lists = Sympa::List::get_lists('*');
688
689
690
691
692
        foreach my $list (@$all_lists) {
            foreach my $f (
                'config',   'info',
                'homepage', 'message.header',
                'message.footer'
Luc Didry's avatar
Luc Didry committed
693
            ) {
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
                if (-f $list->{'dir'} . '/' . $f) {
                    push @files,
                        [$list->{'dir'} . '/' . $f, $list->{'admin'}{'lang'}];
                }
            }

            foreach my $type ('mail_tt2', 'web_tt2', 'scenari') {
                my $directory = $list->{'dir'} . '/' . $type;
                if (-d $directory) {
                    push @directories, [$directory, $list->{'admin'}{'lang'}];
                }
            }
        }

        ## Search language directories
        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }

            if ($d =~ /(mail_tt2|web_tt2)$/) {
                foreach
                    my $subdir (grep(/^[a-z]{2}(_[A-Z]{2})?$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories, ["$d/$subdir", $subdir];
                    }
                }
                closedir DIR;

            } elsif ($d =~ /(create_list_templates|families)$/) {
                foreach my $subdir (grep(/^\w+$/, readdir DIR)) {
                    if (-d "$d/$subdir") {
                        push @directories,
                            ["$d/$subdir", $Conf::Conf{'lang'}];
                    }
                }
                closedir DIR;
            }
        }

        foreach my $pair (@directories) {
            my ($d, $lang) = @$pair;
            unless (opendir DIR, $d) {
                next;
            }
            foreach my $file (readdir DIR) {
                next
                    unless (
                    (   $d =~
                        /mail_tt2|web_tt2|create_list_templates|families/
                        && $file =~ /\.tt2$/
                    )
                    || ($d =~ /scenari$/ && $file =~ /\w+\.\w+$/)
                    );
                push @files, [$d . '/' . $file, $lang];
            }
            closedir DIR;
        }

        ## Do the encoding modifications
        ## Previous versions of files are backed up with the date extension
        my $total = to_utf8(\@files);
757
        $log->syslog('notice', '%d files have been modified', $total);
758
759
    }

760
761
    ## giving up subscribers flat files ; moving subscribers to the DB
    ## Also giving up old 'database' mode
762
    if (lower_version($previous_version, '5.4a.1')) {
763

764
        $log->syslog('notice',
765
766
767
            'Looking for lists with user_data_source parameter set to file or database...'
        );

768
        my $all_lists = Sympa::List::get_lists('*');
769
770
771
772
        foreach my $list (@$all_lists) {

            if ($list->{'admin'}{'user_data_source'} eq 'file') {

773
                $log->syslog(
774
                    'notice',
775
                    'List %s; changing user_data_source from file to include2...',
776
777
778
                    $list->{'name'}
                );

779
780
781
782
                # Load <list dir>/subscribers to the DB
                if (-e $list->{'dir'} . '/subscribers'
                    and rename $list->{'dir'} . '/subscribers',
                    $list->{'dir'} . '/member.dump'
Luc Didry's avatar
Luc Didry committed
783
                ) {
784
                    $list->restore_users('member');
785

786
787
788
                    my $total = $list->{'add_outcome'}{'added_members'};
                    if (defined $list->{'add_outcome'}{'errors'}) {
                        $log->syslog('err', 'Failed to add users: %s',
Luc Didry's avatar
Luc Didry committed
789
790
                            $list->{'add_outcome'}{'errors'}{'error_message'}
                        );
791
792
793
794
                    }
                    $log->syslog('notice',
                        '%d subscribers have been loaded into the database',
                        $total);
795
796
                }

797
                $list->{'admin'}{'user_data_source'} = 'include2';
798
799

                unless ($list->save_config('automatic')) {
800
                    $log->syslog('err',
801
                        'Failed to save config file for list %s', $list);
802
803
804
                }
            } elsif ($list->{'admin'}{'user_data_source'} eq 'database') {

805
                $log->syslog(
806
                    'notice',
807
                    'List %s; changing user_data_source from database to include2...',
808
809
810
811
                    $list->{'name'}
                );

                unless ($list->update_list_member('*', {'subscribed' => 1})) {
812
                    $log->syslog('err',
813
814
815
816
817
818
                        'Failed to update subscribed DB field');
                }

                $list->{'admin'}{'user_data_source'} = 'include2';

                unless ($list->save_config('automatic')) {
819
                    $log->syslog('err',
820
821
822
823
824
                        'Failed to save config file for list %s',
                        $list->{'name'});
                }
            }
        }
825
    }
826

827
    if (lower_version($previous_version, '5.5a.1')) {
sikeda's avatar
sikeda committed
828
        # Remove OTHER/ subdirectories in bounces
829
        $log->syslog('notice', "Removing obsolete OTHER/ bounce directories");
sikeda's avatar
sikeda committed
830
831
        if (opendir my $dh, $Conf::Conf{'bounce_path'}) {
            foreach my $subdir (sort grep (!/^\.+$/, readdir $dh)) {
832
                my $other_dir =
sikeda's avatar
sikeda committed
833
                    $Conf::Conf{'bounce_path'} . '/' . $subdir . '/OTHER';
834
                if (-d $other_dir) {
835
                    Sympa::Tools::File::remove_dir($other_dir);
836
837
                    $log->syslog('notice', 'Directory %s removed',
                        $other_dir);
838
839
                }
            }
sikeda's avatar
sikeda committed
840
            closedir $dh;
841
        } else {
842
            $log->syslog(
843
844
                'err',
                'Failed to open directory %s: %m',
sikeda's avatar
sikeda committed
845
                $Conf::Conf{'bounce_path'}
846
            );
847
848
849
        }
    }

850
    if (lower_version($previous_version, '6.1b.5')) {
851
852
853
854
855
        ## Encoding of shared documents was not consistent with recent
        ## versions of MIME::Encode
        ## MIME::EncWords::encode_mimewords() used to encode characters -!*+/
        ## Now these characters are preserved, according to RFC 2047 section 5
        ## We change encoding of shared documents according to new algorithm
856
        $log->syslog('notice',
857
            'Fixing Q-encoding of web document filenames...');
858
859
860
        system Sympa::Constants::SCRIPTDIR()
            . '/upgrade_shared_repository.pl',
            '--all_lists', '--fix_qencode';
861
    }
862
    if (lower_version($previous_version, '6.1.11')) {
863
        ## Exclusion table was not robot-enabled.
864
        $log->syslog('notice', 'Fixing robot column of exclusion table');
865
        my $sth;
sikeda's avatar
sikeda committed
866
867
868
        my $sdm = Sympa::DatabaseManager->instance;
        unless ($sdm
            and $sth = $sdm->do_query("SELECT * FROM exclusion_table")) {
869
            $log->syslog('err',
870
                'Unable to gather information from the exclusions table');
871
        }
872
        my @robots = Sympa::List::get_robots();
873
874
875
876
877
878
879
880
        while (my $data = $sth->fetchrow_hashref) {
            next
                if (defined $data->{'robot_exclusion'}
                && $data->{'robot_exclusion'} ne '');
            ## Guessing right robot for each exclusion.
            my $valid_robot = '';
            my @valid_robot_candidates;
            foreach my $robot (@robots) {
sikeda's avatar
sikeda committed
881
882
                if (my $list =
                    Sympa::List->new($data->{'list_exclusion'}, $robot)) {
883
884
885
886
887
888
889
890
891
                    if ($list->is_list_member($data->{'user_exclusion'})) {
                        push @valid_robot_candidates, $robot;
                    }
                }
            }
            if ($#valid_robot_candidates == 0) {
                $valid_robot = $valid_robot_candidates[0];
                my $sth;
                unless (
sikeda's avatar
sikeda committed
892
                    $sth = $sdm->do_query(
893
                        "UPDATE exclusion_table SET robot_exclusion = %s WHERE list_exclusion=%s AND user_exclusion=%s",
sikeda's avatar
sikeda committed
894
895
896
                        $sdm->quote($valid_robot),
                        $sdm->quote($data->{'list_exclusion'}),
                        $sdm->quote($data->{'user_exclusion'})
897
                    )
Luc Didry's avatar
Luc Didry committed
898
                ) {
899
                    $log->syslog(
900
                        'err',
901
                        'Unable to update entry (%s, %s) in exclusions table (trying to add robot %s)',
902
903
904
905
906
907
                        $data->{'list_exclusion'},
                        $data->{'user_exclusion'},
                        $valid_robot
                    );
                }
            } else {
908
                $log->syslog(
909
                    'err',
910
                    'Exclusion robot could not be guessed for user "%s" in list "%s". Either this user is no longer subscribed to the list or the list appears in more than one robot (or the query to the database failed). Here is the list of robots in which this list name appears: "%s"',
911
912
913
914
915
916
917
                    $data->{'user_exclusion'},
                    $data->{'list_exclusion'},
                    @valid_robot_candidates
                );
            }
        }
        ## Caching all lists config subset to database
918
        $log->syslog('notice', 'Caching all list config to database...');
919
920
        Sympa::List::_flush_list_db();
        my $all_lists = Sympa::List::get_lists('*', 'reload_config' => 1);
921
922
923
        foreach my $list (@$all_lists) {
            $list->_update_list_db;
        }
924
        $log->syslog('notice', '...done');
925
    }
926

927
    ## We have obsoleted wwsympa.conf.  It would be migrated to sympa.conf.
928
    if (lower_version($previous_version, '6.2b.1')) {
929
930
        $log->syslog('notice', 'Migrating wwsympa.conf...');

931
932
933
934
935
        my $sympa_conf   = Conf::get_sympa_conf();
        my $wwsympa_conf = Conf::get_wwsympa_conf();
        my $fh;
        my %migrated = ();
        my @newconf  = ();
936
        my ($date, $human_date);
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973

        ## Some sympa.conf parameters were overridden by wwsympa.conf.
        ## Others prefer sympa.conf.
        my %wwsconf_override = (
            'arc_path'                   => 'yes',
            'archive_default_index'      => 'yes',
            'bounce_path'                => 'yes',
            'cookie_domain'              => 'NO',
            'cookie_expire'              => 'yes',
            'cookie_refresh'             => 'NO',
            'custom_archiver'            => 'yes',
            'default_home'               => 'NO',
            'export_topics'              => 'yes',
            'html_editor_file'           => 'NO',    # 6.2a
            'html_editor_init'           => 'NO',
            'ldap_force_canonical_email' => 'NO',
            'log_facility'               => 'yes',
            'mhonarc'                    => 'yes',
            'password_case'              => 'NO',
            'review_page_size'           => 'yes',
            'title'                      => 'NO',
            'use_html_editor'            => 'NO',
            'viewlogs_page_size'         => 'yes',
            'wws_path'                   => undef,
        );
        ## Old params
        my %old_param = (
            'alias_manager' => 'No more used, using '
                . $Conf::Conf{'alias_manager'},
            'wws_path' => 'No more used',
            'icons_url' =>
                'No more used. Using static_content/icons instead.',
            'robots' =>
                'Not used anymore. Robots are fully described in their respective robot.conf file.',
            'task_manager_pidfile' => 'No more used',
            'archived_pidfile'     => 'No more used',
            'bounced_pidfile'      => 'No more used',
974
            'use_fast_cgi'         => 'No longer used',   # 6.2.25b deprecated
IKEDA Soji's avatar
IKEDA Soji committed
975
            'htmlarea_url'         => 'No longer used',   # 6.2.36 deprecated
976
977
978
979
        );

        ## Set language of new file content
        $language->push_lang($Conf::Conf{'lang'});
Sympa authors's avatar
Sympa authors committed
980
        $date       = time;
981
982
        $human_date = $language->gettext_strftime('%d %b %Y at %H:%M:%S',
            localtime $date);
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998

        if (-r $wwsympa_conf) {
            ## load only sympa.conf
            my $conf = (
                Conf::_load_config_file_to_hash(
                    {'path_to_config_file' => $sympa_conf}
                    )
                    || {}
            )->{'config'};
            # not yet implemented.
            #my $conf = Conf::load_robot_conf(
            #    {'robot' => '*', 'no_db' => 1, 'return_result' => 1}
            #);

            my %infile = ();
            ## load defaults
999
            foreach my $p (@Sympa::ConfDef::params) {
1000
                next unless $p->{'name'};
For faster browsing, not all history is shown. View entire blame