Commit a68d42a6 authored by sympa-authors's avatar sympa-authors
Browse files

[feature] add support of postgresql in automatic upgrade process


git-svn-id: https://subversion.renater.fr/sympa/trunk@6975 05aa8bb8-cd2b-0410-b1d7-8918dfa770ce
parent ff4545b5
......@@ -12,9 +12,6 @@ CREATE DATABASE sympa;
--
CREATE TABLE conf_table (
robot_conf varchar(80),
label_conf varchar(80),
value_conf varchar(300),
CONSTRAINT ind_conf PRIMARY KEY (robot_conf, label_conf)
);
......@@ -25,17 +22,6 @@ CREATE TABLE conf_table (
--
CREATE TABLE notification_table (
type_notification varchar(15),
arrival_date_notification varchar(80),
status_notification varchar(100),
recipient_notification varchar(100),
message_notification text,
message_id_notification varchar(100),
date_notification int4 NOT NULL,
reception_option_notification varchar(20),
pk_notification serial NOT NULL,
list_notification varchar(50),
robot_notification varchar(80),
CONSTRAINT ind_notification PRIMARY KEY (pk_notification)
);
......@@ -46,19 +32,6 @@ CREATE TABLE notification_table (
--
CREATE TABLE logs_table (
list_logs varchar(50),
parameters_logs varchar(100),
msg_id_logs varchar(255),
action_logs varchar(50) NOT NULL,
client_logs varchar(100),
user_email_logs varchar(100),
daemon_logs varchar(10) NOT NULL,
target_email_logs varchar(100),
status_logs varchar(10) NOT NULL,
error_type_logs varchar(150),
robot_logs varchar(80),
date_logs int4 NOT NULL,
id_logs bigint NOT NULL,
CONSTRAINT ind_logs PRIMARY KEY (id_logs)
);
......@@ -69,16 +42,6 @@ CREATE TABLE logs_table (
--
CREATE TABLE user_table (
email_user varchar(100) NOT NULL,
last_login_date_user int4,
attributes_user varchar(500),
data_user varchar(500),
cookie_delay_user int4,
last_login_host_user varchar(60),
gecos_user varchar(150),
password_user varchar(40),
lang_user varchar(10),
wrong_login_count_user int4,
CONSTRAINT ind_user PRIMARY KEY (email_user)
);
......@@ -89,9 +52,6 @@ CREATE TABLE user_table (
--
CREATE TABLE exclusion_table (
user_exclusion varchar(100) NOT NULL,
date_exclusion int4,
list_exclusion varchar(50) NOT NULL,
CONSTRAINT ind_exclusion PRIMARY KEY (list_exclusion, user_exclusion)
);
......@@ -102,14 +62,6 @@ CREATE TABLE exclusion_table (
--
CREATE TABLE session_table (
data_session varchar(500),
id_session varchar(30) NOT NULL,
remote_addr_session varchar(60),
date_session int4 NOT NULL,
robot_session varchar(80),
hit_session int4,
email_session varchar(100),
start_date_session int4 NOT NULL,
CONSTRAINT ind_session PRIMARY KEY (id_session)
);
......@@ -120,10 +72,6 @@ CREATE TABLE session_table (
--
CREATE TABLE netidmap_table (
netid_netidmap varchar(100) NOT NULL,
email_netidmap varchar(100),
robot_netidmap varchar(80) NOT NULL,
serviceid_netidmap varchar(100) NOT NULL,
CONSTRAINT ind_netidmap PRIMARY KEY (netid_netidmap, serviceid_netidmap, robot_netidmap)
);
......@@ -134,15 +82,6 @@ CREATE TABLE netidmap_table (
--
CREATE TABLE bulkspool_table (
dkim_selector_bulkspool varchar(50),
dkim_d_bulkspool varchar(50),
dkim_privatekey_bulkspool varchar(1000),
messageid_bulkspool varchar(100),
dkim_i_bulkspool varchar(100),
dkim_header_list_bulkspool varchar(500),
messagekey_bulkspool varchar(33) NOT NULL,
message_bulkspool text,
lock_bulkspool int4,
CONSTRAINT ind_bulkspool PRIMARY KEY (messagekey_bulkspool)
);
......@@ -153,20 +92,6 @@ CREATE TABLE bulkspool_table (
--
CREATE TABLE admin_table (
date_admin timestamp with time zone NOT NULL,
update_admin timestamp with time zone,
info_admin varchar(150),
list_admin varchar(50) NOT NULL,
reception_admin varchar(20),
comment_admin varchar(150),
subscribed_admin int4,
profile_admin varchar(15),
role_admin varchar(15) NOT NULL,
included_admin int4,
user_admin varchar(100) NOT NULL,
include_sources_admin varchar(50),
robot_admin varchar(80) NOT NULL,
visibility_admin varchar(20),
CONSTRAINT ind_admin PRIMARY KEY (robot_admin, list_admin, role_admin, user_admin)
);
......@@ -177,13 +102,6 @@ CREATE TABLE admin_table (
--
CREATE TABLE one_time_ticket_table (
data_one_time_ticket varchar(200),
status_one_time_ticket varchar(60),
ticket_one_time_ticket varchar(30),
remote_addr_one_time_ticket varchar(60),
robot_one_time_ticket varchar(80),
date_one_time_ticket int4,
email_one_time_ticket varchar(100),
CONSTRAINT ind_one_time_ticket PRIMARY KEY (ticket_one_time_ticket)
);
......@@ -194,21 +112,6 @@ CREATE TABLE one_time_ticket_table (
--
CREATE TABLE bulkmailer_table (
priority_packet_bulkmailer int4,
receipients_bulkmailer varchar(500),
reception_date_bulkmailer int4,
packetid_bulkmailer varchar(33) NOT NULL,
priority_message_bulkmailer int4,
verp_bulkmailer int4,
merge_bulkmailer int4,
tracking_bulkmailer varchar(15),
returnpath_bulkmailer varchar(100),
messageid_bulkmailer varchar(100),
messagekey_bulkmailer varchar(80) NOT NULL,
listname_bulkmailer varchar(50),
delivery_date_bulkmailer int4,
robot_bulkmailer varchar(80),
lock_bulkmailer varchar(30),
CONSTRAINT ind_bulkmailer PRIMARY KEY (messagekey_bulkmailer, packetid_bulkmailer)
);
......@@ -219,26 +122,6 @@ CREATE TABLE bulkmailer_table (
--
CREATE TABLE subscriber_table (
robot_subscriber varchar(80) NOT NULL,
reception_subscriber varchar(20),
custom_attribute_subscriber varchar(500),
list_subscriber varchar(50) NOT NULL,
visibility_subscriber varchar(20),
comment_subscriber varchar(150),
suspend_start_date_subscriber int4,
suspend_end_date_subscriber int4,
bounce_address_subscriber varchar(100),
bounce_subscriber varchar(35),
included_subscriber int4,
include_sources_subscriber varchar(50),
update_subscriber timestamp with time zone,
user_subscriber varchar(100) NOT NULL,
topics_subscriber varchar(200),
subscribed_subscriber int4,
date_subscriber timestamp with time zone NOT NULL,
bounce_score_subscriber int4,
number_messages_subscriber int4 NOT NULL,
suspend_subscriber int4,
CONSTRAINT ind_subscriber PRIMARY KEY (robot_subscriber, list_subscriber, user_subscriber)
);
......@@ -249,15 +132,5 @@ CREATE TABLE subscriber_table (
--
CREATE TABLE stat_table (
daemon_stat varchar(10),
list_stat varchar(150),
parameter_stat varchar(50),
id_stat bigint NOT NULL,
email_stat varchar(100),
operation_stat varchar(50) NOT NULL,
user_ip_stat varchar(100),
read_stat tinyint(1) NOT NULL,
date_stat int4 NOT NULL,
robot_stat varchar(80) NOT NULL,
CONSTRAINT ind_stat PRIMARY KEY (id_stat)
);
......@@ -204,12 +204,13 @@ sub db_struct {
#Postgresql
$trans_pg =~ s/^int(1)/smallint/g;
$trans_pg =~ s/^int\(.*/int4/g;
$trans_pg =~ s/^bigint.*/bigint/g;
$trans_pg =~ s/^smallint.*/int4/g;
$trans_pg =~ s/^tinyint\(.*\)/int2/g;
$trans_pg =~ s/^bigint.*/int8/g;
$trans_pg =~ s/^enum.*/varchar(15)/g;
$trans_pg =~ s/^text.*/varchar(500)/g;
$trans_pg =~ s/^longtext.*/text/g;
$trans_pg =~ s/^datetime.*/timestamp with time zone/g;
$trans_pg =~ s/^datetime.*/timestamptz/g;
#Sybase
$trans_syb =~ s/^int.*/numeric/g;
$trans_syb =~ s/^text.*/varchar(500)/g;
......@@ -226,7 +227,7 @@ sub db_struct {
$trans_sq =~ s/^datetime.*/timestamp/g;
$trans_sq =~ s/^enum.*/text/g;
$db_struct{'pg'}{$table}{$field} = $trans_pg;
$db_struct{'Pg'}{$table}{$field} = $trans_pg;
$db_struct{'Oracle'}{$table}{$field} = $trans_o;
$db_struct{'Sybase'}{$table}{$field} = $trans_syb;
$db_struct{'SQLite'}{$table}{$field} = $trans_sq;
......
......@@ -40,7 +40,7 @@ my %not_null = %Sympa::Constants::not_null;
my %primary = %Sympa::Constants::primary ;
my %autoincrement = %Sympa::Constants::primary ;
my %autoincrement = %Sympa::Constants::autoincrement ;
## List the required INDEXES
## 1st key is the concerned table
......@@ -127,7 +127,7 @@ sub upgrade {
&do_log('notice','Migrating templates to TT2 format...');
my $tpl_script = Sympa::Constants::SCRIPTDIR . '/tpl2tt2.pl';
my $tpl_script = Sympa::Constants::SCRIPTDIR . '/tpl2tt2.pl';
unless (open EXEC, "$tpl_script|") {
&do_log('err', "Unable to run $tpl_script");
return undef;
......@@ -727,6 +727,82 @@ sub upgrade {
return 1;
}
# return 1 if table.field is autoincrement
sub is_autoinc {
my $table = shift; my $field = shift;
&do_log('debug', 'is_autoinc(%s,%s)',$table,$field);
return undef unless $table;
return undef unless $field;
my $seqname = $table.'_'.$field.'_seq';
my $sth;
my $dbh = &List::db_get_handler();
if ($Conf::Conf{'db_type'} eq 'Pg') {
my $sql_query = "SELECT relname FROM pg_class WHERE relname = '$seqname' AND relkind = 'S' AND relnamespace IN ( SELECT oid FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname != 'information_schema' )";
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
my $autoinc = 0;
my $field = $sth->fetchrow();
$sth->finish();
return ($field eq $seqname);
}else{
do_log('debug',"automatic upgrade : autoincrement for table $table, field $field : test of existing autoinc not yet supported for db_type = $Conf::Conf{'db_type'} ");
return undef;
}
}
# modify table.field as autoincrement
sub set_autoinc {
my $table = shift; my $field = shift;
&do_log('debug', 'set_autoinc(%s,%s)',$table,$field);
return undef unless $table;
return undef unless $field;
my $seqname = $table.'_'.$field.'_seq';
my $sth;
my $dbh = &List::db_get_handler();
if ($Conf::Conf{'db_type'} eq 'Pg') {
my $sql_query = "CREATE SEQUENCE $seqname";
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
$sth->finish();
$sql_query = "ALTER TABLE $table ALTER COLUMN $field SET DEFAULT NEXTVAL('$seqname');";
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
return ;
}else{
do_log('debug',"automatic upgrade : autoincrement for table $table, field $field : test of existing autoinc not yet supported for db_type = $Conf::Conf{'db_type'} ");
return undef;
}
}
sub probe_db {
&do_log('debug3', 'List::probe_db()');
my (%checked, $table);
......@@ -754,28 +830,32 @@ sub probe_db {
}
my $dbh = &List::db_get_handler();
my (@tables, $fields, %real_struct);
my @tables ;
## Get tables
if ($Conf::Conf{'db_type'} eq 'mysql') {
## Get tables
@tables = $dbh->tables();
foreach my $t (@tables) {
$t =~ s/^\`[^\`]+\`\.//;## Clean table names that would look like `databaseName`.`tableName` (mysql)
$t =~ s/^\`(.+)\`$/$1/;## Clean table names that could be surrounded by `` (recent DBD::mysql release)
}
unless (defined $#tables) {
&do_log('info', 'Can\'t load tables list from database %s : %s', $Conf::Conf{'db_name'}, $dbh->errstr);
return undef;
}
}elsif($Conf::Conf{'db_type'} eq 'Pg') {
@tables = $dbh->tables(undef,'public',undef,'TABLE',{pg_noprefix => 1} );
}
unless (defined $#tables) {
&do_log('info', 'Can\'t load tables list from database %s : %s', $Conf::Conf{'db_name'}, $dbh->errstr);
return undef;
}
my ( $fields, %real_struct);
if (($Conf::Conf{'db_type'} eq 'mysql') || ($Conf::Conf{'db_type'} eq 'Pg')){
## Check required tables
foreach my $t1 (keys %{$db_struct{'mysql'}}) {
my $found;
foreach my $t2 (@tables) {
$found = 1 if ($t1 eq $t2);
$found = 1 if ($t1 eq $t2) ;
}
unless ($found) {
unless ($dbh->do("CREATE TABLE $t1 (temporary INT)")) {
......@@ -789,37 +869,35 @@ sub probe_db {
$real_struct{$t1} = {};
}
}
## Get fields
foreach my $t (@tables) {
my $sth;
my $sth;
# unless ($sth = $dbh->table_info) {
# unless ($sth = $dbh->prepare("LISTFIELDS $t")) {
my $sql_query = "SHOW FIELDS FROM $t";
my $sql_query;
if ( $Conf::Conf{'db_type'} eq 'Pg'){
$sql_query = 'SELECT a.attname AS field, t.typname AS type, a.atttypmod AS lengh FROM pg_class c, pg_attribute a, pg_type t WHERE a.attnum > 0 and a.attrelid = c.oid and c.relname = \''.$t.'\' and a.atttypid = t.oid order by a.attnum';
}elsif ($Conf::Conf{'db_type'} eq 'mysql') {
$sql_query = "SHOW FIELDS FROM $t";
}
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
while (my $ref = $sth->fetchrow_hashref('NAME_lc')) {
while (my $ref = $sth->fetchrow_hashref('NAME_lc')) {
$real_struct{$t}{$ref->{'field'}} = $ref->{'type'};
}
if ( $Conf::Conf{'db_type'} eq 'Pg'){
my $lengh = $ref->{'lengh'} - 4; # What a dirty method ! We give a Sympa tee shirt to anyone that suggest a clean solution ;-)
$real_struct{$t}{$ref->{'field'}} = $ref->{'type'}.'('.$lengh.')' if ( $ref->{'type'} eq 'varchar');
}
}
$sth->finish();
}
}elsif ($Conf::Conf{'db_type'} eq 'Pg') {
unless (@tables = $dbh->tables) {
&do_log('err', 'Can\'t load tables list from database %s', $Conf::Conf{'db_name'});
return undef;
}
}elsif ($Conf::Conf{'db_type'} eq 'SQLite') {
unless (@tables = $dbh->tables) {
......@@ -922,6 +1000,7 @@ sub probe_db {
## Check tables structure if we could get it
## Only performed with mysql and SQLite
if (%real_struct) {
foreach my $t (keys %{$db_struct{$Conf::Conf{'db_type'}}}) {
unless ($real_struct{$t}) {
&do_log('err', 'Table \'%s\' not found in database \'%s\' ; you should create it with create_db.%s script', $t, $Conf::Conf{'db_name'}, $Conf::Conf{'db_type'});
......@@ -943,8 +1022,10 @@ sub probe_db {
if ( $autoincrement{$t} eq $f) {
$options .= ' AUTO_INCREMENT PRIMARY KEY ';
}
unless ($dbh->do("ALTER TABLE $t ADD $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options")) {
&do_log('err', 'Could not add field \'%s\' to table\'%s\'.', $f, $t);
my $sqlquery = "ALTER TABLE $t ADD $f $db_struct{$Conf::Conf{'db_type'}}{$t}{$f} $options";
unless ($dbh->do($sqlquery)) {
&do_log('err', 'Could not add field \'%s\' to table\'%s\'. (%s)', $f, $t, $sqlquery);
&do_log('err', 'Sympa\'s database structure may have change since last update ; please check RELEASE_NOTES');
return undef;
}
......@@ -970,8 +1051,8 @@ sub probe_db {
required_format => $db_struct{$Conf::Conf{'db_type'}}{$t}{$f})) {
push @report, sprintf('Field \'%s\' (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...',
$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
&do_log('notice', 'Field \'%s\' (table \'%s\' ; database \'%s\') does NOT have awaited type (%s). Attempting to change it...',
$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f});
&do_log('notice', 'Field \'%s\' (table \'%s\' ; database \'%s\') does NOT have awaited type (%s) where type in database seems to be (%s). Attempting to change it...',
$f, $t, $Conf::Conf{'db_name'}, $db_struct{$Conf::Conf{'db_type'}}{$t}{$f},$real_struct{$t}{$f});
my $options;
if ($not_null{$f}) {
......@@ -997,16 +1078,49 @@ sub probe_db {
}
}
}
if ($Conf::Conf{'db_type'} eq 'mysql') {
if (($Conf::Conf{'db_type'} eq 'mysql')||($Conf::Conf{'db_type'} eq 'Pg')) {
## Check that primary key has the right structure.
my $should_update;
my $test_request_result = $dbh->selectall_hashref('SHOW COLUMNS FROM '.$t,'key');
my %primaryKeyFound;
foreach my $scannedResult ( keys %$test_request_result ) {
if ( $scannedResult eq "PRI" ) {
$primaryKeyFound{$scannedResult} = 1;
my %primaryKeyFound;
my $sql_query ;
my $test_request_result ;
if ($Conf::Conf{'db_type'} eq 'mysql') { # get_primary_keys('mysql');
$sql_query = "SHOW COLUMNS FROM $t";
$test_request_result = $dbh->selectall_hashref($sql_query,'key');
foreach my $scannedResult ( keys %$test_request_result ) {
if ( $scannedResult eq "PRI" ) {
$primaryKeyFound{$scannedResult} = 1;
}
}
}elsif ( $Conf::Conf{'db_type'} eq 'Pg'){# get_primary_keys('Pg');
# $sql_query = "SELECT column_name FROM information_schema.columns WHERE table_name = $t";
# my $sql_query = 'SELECT pg_attribute.attname AS field FROM pg_index, pg_class, pg_attribute WHERE pg_class.oid =\''.$t.'\'::regclass AND indrelid = pg_class.oid AND pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = any(pg_index.indkey) AND indisprimary';
# $test_request_result = $dbh->selectall_hashref($sql_query,'key');
my $sql_query = 'SELECT pg_attribute.attname AS field FROM pg_index, pg_class, pg_attribute WHERE pg_class.oid =\''.$t.'\'::regclass AND indrelid = pg_class.oid AND pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = any(pg_index.indkey) AND indisprimary';
my $sth;
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
while (my $ref = $sth->fetchrow_hashref('NAME_lc')) {
$primaryKeyFound{$ref->{'field'}} = 1;
}
$sth->finish();
}
foreach my $field (@{$primary{$t}}) {
unless ($primaryKeyFound{$field}) {
$should_update = 1;
......@@ -1075,14 +1189,33 @@ sub probe_db {
}
## drop previous index if this index is not a primary key and was defined by a previous Sympa version
$test_request_result = $dbh->selectall_hashref('SHOW INDEX FROM '.$t,'key_name');
#xxxxx $test_request_result = $dbh->selectall_hashref('SHOW INDEX FROM '.$t,'key_name');
my %index_columns;
foreach my $indexName ( keys %$test_request_result ) {
unless ( $indexName eq "PRIMARY" ) {
$index_columns{$indexName} = 1;
if ( $Conf::Conf{'db_type'} eq 'mysql' ){# get_index('Pg');
$test_request_result = $dbh->selectall_hashref('SHOW INDEX FROM '.$t,'key_name');
foreach my $indexName ( keys %$test_request_result ) {
unless ( $indexName eq "PRIMARY" ) {
$index_columns{$indexName} = 1;
}
}
}elsif ( $Conf::Conf{'db_type'} eq 'Pg'){# get_index('Pg');
my $sql_query = 'SELECT pg_attribute.attname AS field FROM pg_index, pg_class, pg_attribute WHERE pg_class.oid =\''.$t.'\'::regclass AND indrelid = pg_class.oid AND pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = any(pg_index.indkey)';
my $sth;
unless ($sth = $dbh->prepare($sql_query)) {
do_log('err','Unable to prepare SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL query %s : %s', $sql_query, $dbh->errstr);
return undef;
}
while (my $ref = $sth->fetchrow_hashref('NAME_lc')) {
$index_columns{$ref->{'field'}} = 1;
}
$sth->finish();
}
foreach my $idx ( keys %index_columns ) {
......@@ -1159,7 +1292,17 @@ sub probe_db {
}
}
}
## Try to run the create_db.XX script
# add autoincrement if needed
foreach my $table (keys %autoincrement) {
unless (&is_autoinc ($table,$autoincrement{$table})){
if (&set_autoinc ($table,$autoincrement{$table})){
&do_log('notice',"Setting table $table field $autoincrement{$table} as autoincrement");
}else{
&do_log('err',"Could not set table $table field $autoincrement{$table} aa autoincrement");
}
}
}
## Try to run the create_db.XX script
}elsif ($found_tables == 0) {
my $db_script =
Sympa::Constants::SCRIPTDIR . "/create_db.$Conf::Conf{'db_type'}";
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment