Commit e797b857 authored by sikeda's avatar sikeda
Browse files

[dev] Refactoring on SDM:

- SDM::probe_db() was moved to Sympa::DatabaseManager::probe_db().
- SDM::data_structure_uptodate() was moved to Conf::data_structure_uptodate().
- Functions specific to database manager were moved from Databasedriver.
- Removed functions never used.
ToDo: Prevent direct access to $SDM::db_source.
ToDo: Careful tests.


git-svn-id: https://subversion.renater.fr/sympa/branches/sympa-6.2-branch@11872 05aa8bb8-cd2b-0410-b1d7-8918dfa770ce
parent 5ff1f1b7
......@@ -31,8 +31,8 @@ use Getopt::Long;
use Conf;
use Sympa::Constants;
use Sympa::DatabaseManager;
use Sympa::List;
use SDM;
my %month_idx = qw(jan 1
fev 2
......@@ -86,7 +86,7 @@ unless (Conf::load()) {
die 'config_error';
}
SDM::probe_db();
Sympa::DatabaseManager::probe_db();
chdir $Conf::Conf{'home'};
......
......@@ -11,9 +11,9 @@ use English qw(-no_match_vars);
use Sympa::Archive;
use Conf;
use Sympa::Constants;
use Sympa::DatabaseManager;
use Log;
use Sympa::Message;
use SDM;
use tools;
unless (Conf::load(Sympa::Constants::CONFIG)) {
......@@ -23,7 +23,7 @@ Log::do_openlog($Conf::Conf{'syslog'}, $Conf::Conf{'log_socket_type'},
'sympa');
if ($Conf::Conf{'db_name'} and $Conf::Conf{'db_type'}) {
unless (SDM::probe_db()) {
unless (Sympa::DatabaseManager::probe_db()) {
die "Sympa can't connect to database";
}
} # to check availabity of Sympa database
......
......@@ -30,8 +30,8 @@ use Getopt::Long;
use Conf;
use Sympa::Constants;
use Sympa::DatabaseManager;
use Sympa::List;
use SDM;
my $sympa_conf_file = Sympa::Constants::CONFIG;
......@@ -42,7 +42,7 @@ unless (Conf::load($sympa_conf_file)) {
## Probe Db if defined
if ($Conf::Conf{'db_name'} and $Conf::Conf{'db_type'}) {
unless (SDM::probe_db()) {
unless (Sympa::DatabaseManager::probe_db()) {
die('Database %s defined in sympa.conf has not the right structure or is unreachable. If you don\'t use any database, comment db_xxx parameters in sympa.conf',
$Conf::Conf{'db_name'}
);
......
......@@ -30,6 +30,7 @@ use English qw(-no_match_vars);
use Conf;
use Sympa::Constants;
use Sympa::DatabaseManager;
use Sympa::Language;
use Sympa::List;
use Log;
......@@ -55,7 +56,7 @@ unless (Conf::load()) {
# Check database connectivity and probe database
#FIXME: Is it required?
unless (SDM::check_db_connect('just_try') and SDM::probe_db()) {
unless (Sympa::DatabaseManager::probe_db()) {
die sprintf
"Database %s defined in sympa.conf has not the right structure or is unreachable. verify db_xxx parameters in sympa.conf\n",
$Conf::Conf{'db_name'};
......
......@@ -964,7 +964,7 @@ $plugins->start if $plugins;
## Check that the data structure is uptodate
## If not, set the web interface to maintenance mode
my $maintenance_mode;
unless (SDM::data_structure_uptodate()) {
unless (Conf::data_structure_uptodate()) {
$maintenance_mode = 1;
Log::do_log('err',
"Web interface set to maintenance mode ; you should run sympa.pl --upgrade"
......@@ -1044,10 +1044,9 @@ while ($query = new_loop()) {
}
 
## If in maintenance mode, check if the data structure is now uptodate
if ($maintenance_mode
&& (SDM::data_structure_uptodate()
&& ($EUID eq (getpwnam(Sympa::Constants::USER))[2]))
) {
if ( $maintenance_mode
and Conf::data_structure_uptodate()
and ($EUID eq (getpwnam(Sympa::Constants::USER))[2])) {
$maintenance_mode = undef;
Log::do_log('notice',
"Data structure seem updated, setting OFF maintenance mode");
......@@ -10575,8 +10574,9 @@ sub do_tracking {
$in{'msgid'}, $in{'list'}, $robot
)
) {
Sympa::Report::reject_report_web('user', 'could_not_get_tracking_info',
{}, $param->{'action'}, $list, $param->{'user'}{'email'}, $robot);
Sympa::Report::reject_report_web('user',
'could_not_get_tracking_info', {}, $param->{'action'}, $list,
$param->{'user'}{'email'}, $robot);
wwslog('err',
"could not get tracking info for message_id $in{'msgid'} and list $in{'list'}"
);
......
......@@ -700,6 +700,40 @@ sub checkfiles_as_root {
return 1;
}
## Check if data structures are uptodate
## If not, no operation should be performed before the upgrade process is run
sub data_structure_uptodate {
my $version_file =
Conf::get_robot_conf('*', 'etc') . '/data_structure.version';
my $data_structure_version;
if (-f $version_file) {
my $fh;
unless (open $fh, '<', $version_file) {
Log::do_log('err', 'Unable to open %s: %m', $version_file);
return undef;
}
while (<$fh>) {
next if /^\s*$/;
next if /^\s*\#/;
chomp;
$data_structure_version = $_;
last;
}
close $fh;
}
if (defined $data_structure_version
and $data_structure_version ne Sympa::Constants::VERSION) {
Log::do_log('err',
"Data structure (%s) is not uptodate for current release (%s)",
$data_structure_version, Sympa::Constants::VERSION);
return 0;
}
return 1;
}
## Check a few files
sub checkfiles {
my $config_err = 0;
......
......@@ -45,6 +45,7 @@ nobase_modules_DATA = \
Sympa/DatabaseDriver/PostgreSQL.pm \
Sympa/DatabaseDriver/SQLite.pm\
Sympa/DatabaseDriver/Sybase.pm \
Sympa/DatabaseManager.pm \
Sympa/Datasource.pm \
Sympa/Family.pm \
Sympa/Fetch.pm \
......@@ -108,6 +109,7 @@ MAN3PM = \
Sympa/DatabaseDriver/PostgreSQL.pm \
Sympa/DatabaseDriver/SQLite.pm\
Sympa/DatabaseDriver/Sybase.pm \
Sympa/DatabaseManager.pm \
Sympa/HTMLSanitizer.pm \
Sympa/Language.pm \
Sympa/ListDef.pm \
......
<
......@@ -28,31 +28,8 @@ use strict;
use warnings;
use Conf;
use Sympa::Constants;
use Sympa::Database;
use Sympa::DatabaseDescription;
use Log;
#use Sympa::List;
use tools;
use Sympa::Tools::Data;
# db structure description has moved in Sympa/Constant.pm
my %db_struct = Sympa::DatabaseDescription::db_struct();
my %not_null = Sympa::DatabaseDescription::not_null();
my %primary = Sympa::DatabaseDescription::primary();
my %autoincrement = Sympa::DatabaseDescription::autoincrement();
## List the required INDEXES
## 1st key is the concerned table
## 2nd key is the index name
## the table lists the field on which the index applies
my %indexes = %Sympa::DatabaseDescription::indexes;
# table indexes that can be removed during upgrade process
my @former_indexes = @Sympa::DatabaseDescription::former_indexes;
our $db_source;
......@@ -166,550 +143,11 @@ sub db_disconnect {
return 1;
}
sub probe_db {
Log::do_log('debug3', 'Checking database structure');
my (%checked, $table);
my $db_type = Conf::get_robot_conf('*', 'db_type');
my $update_db_field_types =
Conf::get_robot_conf('*', 'update_db_field_types') || 'off';
unless (check_db_connect()) {
Log::do_log('err',
'Could not check the database structure. Make sure that database connection is available'
);
return undef;
}
# Does the driver support probing database structure?
foreach my $method (
qw(is_autoinc get_tables get_fields get_primary_key get_indexes)) {
unless ($db_source->can($method)) {
Log::do_log('notice',
'Could not check the database structure: required methods have not been implemented'
);
return 1;
}
}
# Does the driver support updating database structure?
my $may_update;
unless ($update_db_field_types eq 'auto') {
$may_update = 0;
} else {
$may_update = 1;
foreach my $method (
qw(set_autoinc add_table update_field add_field delete_field
unset_primary_key set_primary_key unset_index set_index)
) {
unless ($db_source->can($method)) {
$may_update = 0;
last;
}
}
}
## Database structure
## Report changes to listmaster
my @report;
## Get tables
my @tables;
my $list_of_tables;
if ($list_of_tables = $db_source->get_tables()) {
@tables = @{$list_of_tables};
} else {
@tables = ();
}
my ($fields, %real_struct);
## Check required tables
foreach my $t1 (keys %{$db_struct{'mysql'}}) {
my $found;
foreach my $t2 (@tables) {
$found = 1 if ($t1 eq $t2);
}
unless ($found) {
my $rep;
if ( $may_update
and $rep = $db_source->add_table({'table' => $t1})) {
push @report, $rep;
Log::do_log(
'notice', 'Table %s created in database %s',
$t1, Conf::get_robot_conf('*', 'db_name')
);
push @tables, $t1;
$real_struct{$t1} = {};
}
}
}
## Get fields
foreach my $t (keys %{$db_struct{'mysql'}}) {
$real_struct{$t} = $db_source->get_fields({'table' => $t});
}
## Check tables structure if we could get it
## Only performed with mysql , Pg and SQLite
if (%real_struct) {
foreach my $t (keys %{$db_struct{'mysql'}}) {
unless ($real_struct{$t}) {
Log::do_log(
'err',
'Table "%s" not found in database "%s"; you should create it with create_db.%s script',
$t,
Conf::get_robot_conf('*', 'db_name'),
$db_type
);
return undef;
}
unless (
check_fields(
{ 'table' => $t,
'report' => \@report,
'real_struct' => \%real_struct,
'may_update' => $may_update,
}
)
) {
Log::do_log(
'err',
'Unable to check the validity of fields definition for table %s. Aborting',
$t
);
return undef;
}
## Remove temporary DB field
if ($may_update and $real_struct{$t}{'temporary'}) {
$db_source->delete_field(
{ 'table' => $t,
'field' => 'temporary',
}
);
delete $real_struct{$t}{'temporary'};
}
## Check that primary key has the right structure.
unless (
check_primary_key(
{ 'table' => $t,
'report' => \@report,
'may_update' => $may_update
}
)
) {
Log::do_log(
'err',
'Unable to check the validity of primary key for table %s. Aborting',
$t
);
return undef;
}
unless (
check_indexes(
{ 'table' => $t,
'report' => \@report,
'may_update' => $may_update
}
)
) {
Log::do_log(
'err',
'Unable to check the valifity of indexes for table %s. Aborting',
$t
);
return undef;
}
}
# add autoincrement if needed
foreach my $table (keys %autoincrement) {
unless (
$db_source->is_autoinc(
{'table' => $table, 'field' => $autoincrement{$table}}
)
) {
if ($may_update
and $db_source->set_autoinc(
{ 'table' => $table,
'field' => $autoincrement{$table},
'field_type' => $db_struct{$db_type}->{$table}
->{$autoincrement{$table}},
}
)
) {
Log::do_log('notice',
"Setting table $table field $autoincrement{$table} as autoincrement"
);
} else {
Log::do_log('err',
"Could not set table $table field $autoincrement{$table} as autoincrement"
);
return undef;
}
}
}
} else {
Log::do_log('err',
"Could not check the database structure. consider verify it manually before launching Sympa."
);
return undef;
}
## Notify listmaster
tools::send_notify_to_listmaster('*', 'db_struct_updated',
{'report' => \@report})
if @report;
return 1;
}
sub check_fields {
my $param = shift;
my $t = $param->{'table'};
my %real_struct = %{$param->{'real_struct'}};
my $report_ref = $param->{'report'};
my $may_update = $param->{'may_update'};
my $db_type = Conf::get_robot_conf('*', 'db_type');
foreach my $f (sort keys %{$db_struct{$db_type}{$t}}) {
unless ($real_struct{$t}{$f}) {
push @{$report_ref},
sprintf(
"Field '%s' (table '%s' ; database '%s') was NOT found. Attempting to add it...",
$f, $t, Conf::get_robot_conf('*', 'db_name'));
Log::do_log(
'notice',
'Field "%s" (table "%s"; database "%s") was NOT found. Attempting to add it...',
$f,
$t,
Conf::get_robot_conf('*', 'db_name')
);
my $rep;
if ($may_update
and $rep = $db_source->add_field(
{ 'table' => $t,
'field' => $f,
'type' => $db_struct{$db_type}{$t}{$f},
'notnull' => $not_null{$f},
'autoinc' =>
($autoincrement{$t} and $autoincrement{$t} eq $f),
'primary' => (
scalar @{$primary{$t} || []} == 1
and $primary{$t}->[0] eq $f
),
}
)
) {
push @{$report_ref}, $rep;
} else {
Log::do_log('err',
'Addition of fields in database failed. Aborting');
return undef;
}
next;
}
## Change DB types if different and if update_db_types enabled
if ($may_update) {
unless (
check_db_field_type(
effective_format => $real_struct{$t}{$f},
required_format => $db_struct{$db_type}{$t}{$f}
)
) {
push @{$report_ref},
sprintf(
"Field '%s' (table '%s' ; database '%s') does NOT have awaited type (%s). Attempting to change it...",
$f, $t,
Conf::get_robot_conf('*', 'db_name'),
$db_struct{$db_type}{$t}{$f}
);
Log::do_log(
'notice',
'Field "%s" (table "%s"; database "%s") does NOT have awaited type (%s) where type in database seems to be (%s). Attempting to change it...',
$f,
$t,
Conf::get_robot_conf('*', 'db_name'),
$db_struct{$db_type}{$t}{$f},
$real_struct{$t}{$f}
);
my $rep;
if ($may_update
and $rep = $db_source->update_field(
{ 'table' => $t,
'field' => $f,
'type' => $db_struct{$db_type}{$t}{$f},
'notnull' => $not_null{$f},
}
)
) {
push @{$report_ref}, $rep;
} else {
Log::do_log('err',
'Fields update in database failed. Aborting');
return undef;
}
}
} else {
unless ($real_struct{$t}{$f} eq $db_struct{$db_type}{$t}{$f}) {
Log::do_log(
'err',
'Field "%s" (table "%s"; database "%s") does NOT have awaited type (%s)',
$f,
$t,
Conf::get_robot_conf('*', 'db_name'),
$db_struct{$db_type}{$t}{$f}
);
Log::do_log('err',
'Sympa\'s database structure may have change since last update ; please check RELEASE_NOTES'
);
return undef;
}
}
}
return 1;
}
sub check_primary_key {
my $param = shift;
my $t = $param->{'table'};
my $report_ref = $param->{'report'};
my $may_update = $param->{'may_update'};
my $list_of_keys = join ',', @{$primary{$t}};
my $key_as_string = "$t [$list_of_keys]";
Log::do_log('debug',
'Checking primary keys for table %s expected_keys %s',
$t, $key_as_string);
my $should_update = $db_source->check_key(
{ 'table' => $t,
'key_name' => 'primary',
'expected_keys' => $primary{$t}
}
);
if ($should_update) {
my $list_of_keys = join ',', @{$primary{$t}};
my $key_as_string = "$t [$list_of_keys]";
if ($should_update->{'empty'}) {
Log::do_log('notice', 'Primary key %s is missing. Adding it',
$key_as_string);
## Add primary key
my $rep = undef;
if ($may_update
and $rep = $db_source->set_primary_key(
{'table' => $t, 'fields' => $primary{$t}}
)
) {
push @{$report_ref}, $rep;
} else {
return undef;
}
} elsif ($should_update->{'existing_key_correct'}) {
Log::do_log('debug',
"Existing key correct (%s) nothing to change",
$key_as_string);
} else {
## drop previous primary key
my $rep = undef;
if ( $may_update
and $rep = $db_source->unset_primary_key({'table' => $t})) {
push @{$report_ref}, $rep;
} else {
return undef;
}
## Add primary key
$rep = undef;
if ($may_update
and $rep = $db_source->set_primary_key(
{'table' => $t, 'fields' => $primary{$t}}
)
) {
push @{$report_ref}, $rep;
} else {
return undef;
}
}
} else {
Log::do_log('err', 'Unable to evaluate table %s primary key', $t);
return undef;
}
return 1;
}
sub check_indexes {
my $param = shift;
my $t = $param->{'table'};
my $report_ref = $param->{'report'};
my $may_update = $param->{'may_update'};
Log::do_log('debug', 'Checking indexes for table %s', $t);
## drop previous index if this index is not a primary key and was defined
## by a previous Sympa version
my %index_columns = %{$db_source->get_indexes({'table' => $t})};
foreach my $idx (keys %index_columns) {
Log::do_log('debug', 'Found index %s', $idx);
## Remove the index if obsolete.
foreach my $known_index (@former_indexes) {
if ($idx eq $known_index) {
my $rep;
Log::do_log('notice', 'Removing obsolete index %s', $idx);
if ( $may_update
and $rep =
$db_source->unset_index({'table' => $t, 'index' => $idx}))
{
push @{$report_ref}, $rep;
}
last;
}
}
}
## Create required indexes
foreach my $idx (keys %{$indexes{$t}}) {
## Add indexes
unless ($index_columns{$idx}) {
my $rep;
Log::do_log('notice',
'Index %s on table %s does not exist. Adding it',
$idx, $t);
if ($may_update
and $rep = $db_source->set_index(
{ 'table' => $t,
'index_name' => $idx,
'fields' => $indexes{$t}{$idx}
}
)
) {