Commit 9adfefd0 authored by sympa-authors's avatar sympa-authors
Browse files

[Adrien Brard] New feature: DB logs

Each operations theat changes the status of messages/subscriptions/list config is now logged in a structured DB entry. Both listmasters and listowners can search this events DB using the Sympa web interface.


git-svn-id: https://subversion.renater.fr/sympa/trunk@3785 05aa8bb8-cd2b-0410-b1d7-8918dfa770ce
parent 1a3ab0b9
This diff is collapsed.
This diff is collapsed.
......@@ -46,7 +46,7 @@ my @valid_options = qw(
spool queue queuedistribute queueauth queuetask queuebounce queuedigest
queuemod queuetopic queuesubscribe queueoutgoing tmpdir
loop_command_max loop_command_sampling_delay loop_command_decrease_factor loop_prevention_regex
purge_user_table_task purge_orphan_bounces_task eval_bouncers_task process_bouncers_task
purge_user_table_task purge_logs_table_task purge_orphan_bounces_task eval_bouncers_task process_bouncers_task
minimum_bouncing_count minimum_bouncing_period bounce_delay
default_bounce_level1_rate default_bounce_level2_rate
remind_return_path request_priority return_path_suffix rfc2369_header_fields sendmail sendmail_args sleep
......@@ -190,6 +190,8 @@ my %Default_Conf =
'list_check_suffixes' => 'request,owner,editor,unsubscribe,subscribe',
'expire_bounce_task' => 'daily',
'purge_user_table_task' => 'monthly',
'purge_logs_table_task' => 'daily',
'logs_expiration_period' => 3, #3 months
'purge_orphan_bounces_task' => 'monthly',
'eval_bouncers_task' => 'daily',
'process_bouncers_task' => 'weekly',
......
......@@ -10781,6 +10781,20 @@ sub probe_db {
'serviceid_netidmap' => 'varchar(100)',
'email_netidmap' => 'varchar(100)',
'robot_netidmap' => 'varchar(80)'},
'logs_table' => {'id_logs' => 'bigint(20)',
'date_logs' => 'int(11)',
'robot_logs' => 'varchar(80)',
'list_logs' => 'varchar(50)',
'action_logs' => 'varchar(50)',
'parameters_logs' => 'varchar(100)',
'target_email_logs' => 'varchar(100)',
'user_email_logs' => 'varchar(100)',
'msg_id_logs' => 'varchar(255)',
'status_logs' => 'varchar(10)',
'error_type_logs' => 'varchar(150)',
'client_logs' => 'varchar(100)',
'daemon_logs' => 'varchar(10)'
},
},
'SQLite' => {'user_table' => {'email_user' => 'varchar(100)',
'gecos_user' => 'varchar(150)',
......@@ -10820,6 +10834,21 @@ sub probe_db {
'serviceid_netidmap' => 'varchar(100)',
'email_netidmap' => 'varchar(100)',
'robot_netidmap' => 'varchar(80)'},
'logs_table' => {'id_logs' => 'integer',
'date_logs' => 'integer',
'robot_logs' => 'varchar(80)',
'list_logs' => 'varchar(50)',
'action_logs' => 'varchar(50)',
'parameters_logs' => 'varchar(100)',
'target_email_logs' => 'varchar(100)',
'user_email_logs' => 'varchar(100)',
'msg_id_logs' => 'varchar(255)',
'status_logs' => 'varchar(10)',
'error_type_logs' => 'varchar(150)',
'client_logs' => 'varchar(100)',
'daemon_logs' => 'varchar(10)'
},
},
);
......@@ -10835,13 +10864,19 @@ sub probe_db {
'date_admin' => 1,
'netid_netidmap' => 1,
'serviceid_netidmap' => 1,
'robot_netidmap' => 1
'robot_netidmap' => 1,
'id_logs' => 1,
'date_logs' => 1,
'action_logs' => 1,
'status_logs' => 1,
'daemon_logs' => 1
);
my %primary = ('user_table' => ['email_user'],
'subscriber_table' => ['list_subscriber','robot_subscriber','user_subscriber'],
'admin_table' => ['list_admin','robot_admin','user_admin','role_admin'],
'netidmap_table' => ['netid_netidmap','serviceid_netidmap','robot_netidmap']
'netidmap_table' => ['netid_netidmap','serviceid_netidmap','robot_netidmap'],
'logs_table' => ['id_logs']
);
## Report changes to listmaster
......@@ -13249,112 +13284,6 @@ sub remove_task {
return 1;
}
# add log in RDBMS
sub db_log {
my $process = shift;
my $email_user = shift; $email_user = lc($email_user);
my $auth = shift;
my $ip = shift; $ip = lc($ip);
my $ope = shift; $ope = lc($ope);
my $list = shift; $list = lc($list);
my $robot = shift; $robot = lc($robot);
my $arg = shift;
my $status = shift;
my $subscriber_count = shift;
do_log ('info',"db_log (PROCESS = $process, EMAIL = $email_user, AUTH = $auth, IP = $ip, OPERATION = $ope, LIST = $list,ROBOT = $robot, ARG = $arg ,STATUS = $status , LIST= list_subscriber)");
unless ($process =~ /^((task)|(archived)|(sympa)|(wwsympa)|(bounce))$/) {
do_log ('err',"Internal_error : incorrect process value $process");
return undef;
}
unless ($auth =~ /^((smtp)|(md5)|(smime)|(null))$/) {
do_log ('err',"Internal_error : incorrect auth value $auth");
return undef;
}
$auth = '' if ($auth eq 'null');
my $date=time;
## Insert in log_table
my $statement = 'INSERT INTO log_table (id, date, pid, process, email_user, auth, ip, operation, list, robot, arg, status, subscriber_count) ';
my $statement_value = sprintf "VALUES ('',%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", $date,$$,$dbh->quote($process),$dbh->quote($email_user),$dbh->quote($auth),$dbh->quote($ip),$dbh->quote($ope),$dbh->quote($list),$dbh->quote($robot),$dbh->quote($arg),$dbh->quote($status),$subscriber_count;
$statement = $statement.$statement_value;
unless ($dbh->do($statement)) {
do_log('err','Unable to execute SQL statement \n"%s" \n %s', $statement, $dbh->errstr);
return undef;
}
}
# Scan log_table with appropriate select
sub get_first_db_log {
my $select = shift;
do_log('info','get_first_db_log (%s)',$select);
## Check database connection
unless ($dbh and $dbh->ping) {
return undef unless &db_connect();
}
my $statement;
if ($Conf{'db_type'} eq 'Oracle') {
## "AS" not supported by Oracle
$statement = "SELECT date \"date\", pid \"pid\", process \"process\", email_user \"email\", auth \"auth\", ip \"ip\",operation \"operation\", list \"list\", robot \"robot\", arg \"arg\", status \"status\", subscriber_count \"count\" FROM log_table WHERE 1 ";
}else{
$statement = "SELECT date AS date, pid AS pid, process AS process, email_user AS email, auth AS auth, ip AS ip, operation AS operation, list AS list, robot AS robot, arg AS arg, status AS status, subscriber_count AS count FROM log_table WHERE 1 ";
}
if ($select->{'list'}) {
$select->{'list'} = lc ($select->{'list'});
$statement .= sprintf "AND list = %s ",$select->{'list'};
}
if ($select->{'robot'}) {
$select->{'robot'} = lc ($select->{'robot'});
$statement .= sprintf "AND robot = %s ",$select->{'robot'};
}
if ($select->{'ip'}) {
$select->{'ip'} = lc ($select->{'ip'});
$statement .= sprintf "AND ip = %s ",$select->{'ip'};
}
if ($select->{'ope'}) {
$select->{'ope'} = lc ($select->{'ope'});
$statement .= sprintf "AND operation = %s ",$select->{'operation'};
}
push @sth_stack, $sth;
unless ($sth = $dbh->prepare($statement)) {
do_log('err','Unable to prepare SQL statement : %s', $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL statement "%s" : %s', $statement, $dbh->errstr);
return undef;
}
return ($sth->fetchrow_hashref);
}
sub get_next_db_log {
my $log = $sth->fetchrow_hashref;
unless (defined $log) {
$sth->finish;
$sth = pop @sth_stack;
}
return $log;
}
## Close the list (remove from DB, remove aliases, change status to 'closed' or 'family_closed')
sub close {
my ($self, $email, $status) = @_;
......
......@@ -26,11 +26,15 @@ package Log;
require Exporter;
use Sys::Syslog;
use Carp;
use POSIX qw/mktime/;
@ISA = qw(Exporter);
@EXPORT = qw(fatal_err do_log do_openlog $log_level);
my ($log_facility, $log_socket_type, $log_service);
my ($log_facility, $log_socket_type, $log_service,$sth,@sth_stack,$rows_nb);
local $log_level |= 0;
sub fatal_err {
......@@ -108,6 +112,338 @@ sub do_connect {
openlog("$log_service\[$$\]", 'ndelay', $log_facility);
}
# return the name of the used daemon
sub set_daemon {
my $daemon_tmp = shift;
my @path = split(/\//, $daemon_tmp);
my $daemon = $path[$#path];
$daemon =~ s/(\.[^\.]+)$//;
return $daemon;
}
sub get_log_date {
my $date_from,
my $date_to;
my $dbh = &List::db_get_handler();
## Check database connection
unless ($dbh and $dbh->ping) {
return undef unless &db_connect();
}
my $statement;
my @dates;
foreach my $query('MIN','MAX') {
$statement = "SELECT $query(date_logs) FROM logs_table";
push @sth_stack, $sth;
unless($sth = $dbh->prepare($statement)) {
do_log('err','Get_log_date: Unable to prepare SQL statement : %s %s',$statement, $dbh->errstr);
return undef;
}
unless($sth->execute) {
do_log('err','Get_log_date: Unable to execute SQL statement %s %s',$statement, $dbh->errstr);
return undef;
}
while (my $d = ($sth->fetchrow_array) [0]) {
push @dates, $d;
}
}
$sth->finish();
$sth = pop @sth_stack;
return @dates;
}
# add log in RDBMS
sub db_log {
my $arg = shift;
my $list = $arg->{'list'};
my $robot = $arg->{'robot'};
my $action = $arg->{'action'};
my $parameters = &tools::clean_msg_id($arg->{'parameters'});
my $target_email = $arg->{'target_email'};
my $msg_id = &tools::clean_msg_id($arg->{'msg_id'});
my $status = $arg->{'status'};
my $error_type = $arg->{'error_type'};
my $user_email = &tools::clean_msg_id($arg->{'user_email'});
my $client = $arg->{'client'};
my $daemon = $arg->{'daemon'};
my $date=time;
my $random = int(rand(1000));
my $id = $date*1000+$random;
unless($user_email) {
$user_email = 'anonymous';
}
unless($list) {
$list = '';
}
#remove the robot name of the list name
if($list =~ /(.+)\@(.+)/) {
$list = $1;
unless($robot) {
$robot = $2;
}
}
my $dbh = &List::db_get_handler();
## Check database connection
unless ($dbh and $dbh->ping) {
return undef unless &db_connect();
}
unless ($daemon =~ /^((task)|(archived)|(sympa)|(wwsympa)|(bounced)|(sympa_soap))$/) {
do_log ('err',"Internal_error : incorrect process value $daemon");
return undef;
}
## Insert in log_table
my $statement = sprintf 'INSERT INTO logs_table (id_logs,date_logs,robot_logs,list_logs,action_logs,parameters_logs,target_email_logs,msg_id_logs,status_logs,error_type_logs,user_email_logs,client_logs,daemon_logs) VALUES (%s,%d,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
$id,
$date,
$dbh->quote($robot),
$dbh->quote($list),
$dbh->quote($action),
$dbh->quote($parameters),
$dbh->quote($target_email),
$dbh->quote($msg_id),
$dbh->quote($status),
$dbh->quote($error_type),
$dbh->quote($user_email),
$dbh->quote($client),
$dbh->quote($daemon);
unless ($dbh->do($statement)) {
do_log('err','Unable to execute SQL statement "%s", %s', $statement, $dbh->errstr);
return undef;
}
}
# delete logs in RDBMS
sub db_log_del {
my $exp = &Conf::get_robot_conf($Conf{'host'},'logs_expiration_period');
my $date = time - ($exp * 24 * 60 * 60);
my $dbh = &List::db_get_handler();
my $statement = sprintf "DELETE FROM logs_table WHERE (logs_table.date_logs <= %s)", $dbh->quote($date);
unless ($dbh->do($statement)) {
&do_log('err','Unable to execute SQL statement "%s" : %s',$statement, $dbh->errstr);
return undef;
}
}
# Scan log_table with appropriate select
sub get_first_db_log {
my $dbh = &List::db_get_handler();
my $select = shift;
## Dump vars
#open TMP, ">/tmp/logs.dump";
#&tools::dump_var($select, 0, \*TMP);
#close TMP;
my %action_type = ('message' => ['reject','distribute','arc_delete','arc_download',
'sendMessage','remove','record_email','send_me',
'd_remove_arc','rebuildarc','remind','send_mail',
'DoFile','sendMessage','DoForward','DoMessage',
'DoCommand','SendDigest'],
'authentication' => ['login','logout','loginrequest','sendpasswd',
'ssologin','ssologin_succeses','remindpasswd',
'change_identity','choosepasswd'],
'subscription' => ['subscribe','signoff','add','del','ignoresub',
'subindex'],
'list_management' => ['create_list','rename_list','close_list',
'edit_list','admin','blacklist','install_pending_list',
'purge_list','edit_template','copy_template',
'remove_template'],
'bounced' => ['resetbounce','get_bounce'],
'preferences' => ['set','setpref','pref','change_email','setpasswd','record_email','editsubscriber'],
'shared' => ['d_unzip','d_upload','d_read','d_delete','d_savefile',
'd_overwrite','d_create_dir','d_set_owner','d_change_access',
'd_describe','d_rename','d_editfile','d_admin',
'd_install_shared','d_reject_shared','d_properties',
'creation_shared_file','d_unzip_shared_file',
'install_file_hierarchy','d_copy_rec_dir','d_copy_file',
'change_email','set_lang','new_d_read','d_control'],
);
## Check database connection
unless ($dbh and $dbh->ping) {
return undef unless &db_connect();
}
my $statement = sprintf "SELECT date_logs AS date, robot_logs AS robot, list_logs AS list, action_logs AS action, parameters_logs AS parameters, target_email_logs AS target_email,msg_id_logs AS msg_id, status_logs AS status, error_type_logs AS error_type, user_email_logs AS user_email, client_logs AS client, daemon_logs AS daemon FROM logs_table WHERE robot_logs=%s ", $dbh->quote($select->{'robot'});
#If the checkbox is checked, the other parameters of research are not taken into account.
unless ($select->{'all'} eq 'on') {
my $list_selected = 'false'; #like $robot_selected, for a list.
#if a type of target and a target are specified
if (($select->{'type_target'}) && ($select->{'type_target'} ne 'none')) {
if($select->{'target'}) {
$select->{'type_target'} = lc ($select->{'type_target'});
$select->{'target'} = lc ($select->{'target'});
$statement .= 'AND ' . $select->{'type_target'} . '_logs = ' . $dbh->quote($select->{'target'}).' ';
}
}
#if the search is between two date
if ($select->{'date_from'}) {
my @tab_date_from = split(/\//,$select->{'date_from'});
my $date_from = &POSIX::mktime(0,0,-1,$tab_date_from[0],$tab_date_from[1]-1,$tab_date_from[2]-1900);
unless($select->{'date_to'}) {
my $date_from2 = &POSIX::mktime(0,0,25,$tab_date_from[0],$tab_date_from[1]-1,$tab_date_from[2]-1900);
$statement .= sprintf "AND date_logs BETWEEN '%s' AND '%s' ",$date_from, $date_from2;
}
if($select->{'date_to'}) {
my @tab_date_to = split(/\//,$select->{'date_to'});
my $date_to = &POSIX::mktime(0,0,25,$tab_date_to[0],$tab_date_to[1]-1,$tab_date_to[2]-1900);
$statement .= sprintf "AND date_logs BETWEEN '%s' AND '%s' ",$date_from, $date_to;
}
}
#if the search is on a precise type
if ($select->{'type'}) {
if(($select->{'type'} ne 'none') && ($select->{'type'} ne 'all_actions')) {
my $first = 'false';
foreach my $type(@{$action_type{$select->{'type'}}}) {
if($first eq 'false') {
#if it is the first action, put AND on the statement
$statement .= sprintf "AND (logs_table.action_logs = '%s' ",$type;
$first = 'true';
}
#else, put OR
else {
$statement .= sprintf "OR logs_table.action_logs = '%s' ",$type;
}
}
$statement .= ')';
}
}
#if the listmaster want to make a search by an IP adress.
if($select->{'ip'}) {
$statement .= sprintf "AND client_logs = '%s'",$select->{'ip'};
}
## Currently not used
#if the search is on the actor of the action
if ($select->{'user_email'}) {
$select->{'user_email'} = lc ($select->{'user_email'});
$statement .= sprintf "AND user_email_logs = '%s' ",$select->{'user_email'};
}
#if a list is specified -just for owner or above-
if($select->{'list'}) {
if($select->{'list'} eq 'all_list') {
my $first = 'false';
my $which_list;
if($select->{'privilege'} eq 'owner') {
$which_list = 'alllists_owner';
}
if($select->{'privilege'} eq 'listmaster') {
$statement .= sprintf "AND (list_logs = '' ";
$which_list = 'alllists';
$first = 'true';
}
foreach my $alllists(@{$select->{$which_list}}) {
#if it is the first list, put AND on the statement
if($first eq 'false') {
$statement .= sprintf "AND (list_logs = '%s' ",$alllists;
$first = 'true';
}
#else, put OR
else {
$statement .= sprintf "OR list_logs = '%s' ",$alllists;
}
}
$statement .= sprintf ")";
$list_selected = 'true';
}
else {
if($select->{'list'} ne 'none') {
$select->{'list'} = lc ($select->{'list'});
$statement .= sprintf "AND list_logs = '%s' ",$select->{'list'};
$list_selected = 'true';
}
}
}
}
#If the checkbox is checked
else {
#an editor has just an access on the current list and robot
if($select->{'privilege'} eq 'editor') {
$statement .= sprintf "AND list_logs = '%s' ",$select->{'current_list'};
}
if($select->{'privilege'} eq 'owner') {
my $first = 'false';
foreach my $alllists(@{$select->{'alllists_owner'}}) {
#if it is the first list, put AND on the statement
if($first eq 'false') {
$statement .= sprintf "AND (list_logs = '%s' ",$alllists;
$first = 'true';
}
#else, put OR
else {
$statement .= sprintf "OR list_logs = '%s' ",$alllists;
}
}
$statement .= sprintf ")";
}
#on the current robot to the listmaster
if($select->{'privilege'} eq 'listmaster') {
$statement .= sprintf "AND robot_logs = '%s' ",$select->{'robot'};
}
}
$statement .= sprintf "GROUP BY date_logs ";
push @sth_stack, $sth;
unless ($sth = $dbh->prepare($statement)) {
do_log('err','Unable to prepare SQL statement : %s', $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL statement "%s" : %s', $statement, $dbh->errstr);
return undef;
}
$rows_nb = $sth->rows;
## If no rows returned, return an empty hash
## Required to differenciate errors and empty results
if ($rows_nb == 0) {
return {};
}
return ($sth->fetchrow_hashref);
}
sub return_rows_nb {
return $rows_nb;
}
sub get_next_db_log {
my $log = $sth->fetchrow_hashref;
unless (defined $log) {
$sth->finish;
$sth = pop @sth_stack;
}
return $log;
}
1;
......
title.gettext daily delete logs older than 3 months
/ACTION
purge_logs_table ()
next ([execution_date]+1d, ACTION)
......@@ -60,11 +60,28 @@ CREATE TABLE admin_table (
);
CREATE TABLE netidmap_table (
netid_netidmap varchar (100) NOT NULL,
idp_netidmap varchar (100) NOT NULL,
robot_netidmap varchar (80) NOT NULL,
email_netidmap varchar (100),
netid_netidmap varchar2 (100) NOT NULL,
idp_netidmap varchar2 (100) NOT NULL,
robot_netidmap varchar2 (80) NOT NULL,
email_netidmap varchar2 (100),
CONSTRAINT ind_netidmap PRIMARY KEY (netid_netidmap, idp_netidmap, robot_netidmap)
);
CREATE TABLE logs_table (
id_logs number NOT NULL,
date_logs number NOT NULL,
robot_logs varchar2 (80),
list_logs varchar2 (50),
action_logs varchar2 (50) NOT NULL,
parameters_logs varchar2 (100),
target_email_logs varchar2 (100),
user_email_logs varchar2 (100),
msg_id_logs varchar2 (255),
status_logs varchar2 (10) NOT NULL,
error_type_logs varchar2 (150),
client_logs varchar2 (100),
daemon_logs varchar2 (10) NOT NULL,
CONSTRAINT ind_admin PRIMARY KEY (id_logs)
);
!
......@@ -65,3 +65,22 @@ CREATE TABLE netidmap_table (
CONSTRAINT ind_netidmap PRIMARY KEY (netid_netidmap, idp_netidmap, robot_netidmap)
);
CREATE INDEX netidmap_idx ON netidmap_table(netid_netidmap, idp_netidmap, robot_netidmap);
DROP TABLE logs_table;
CREATE TABLE logs_table (
id_logs bigint NOT NULL,
date_logs int4 NOT NULL,
robot_logs varchar (80),
list_logs varchar (50),
action_logs varchar (50) NOT NULL,
parameters_logs varchar (100),
target_email_logs varchar (100),
user_email_logs varchar (100),
msg_id_logs varchar (255),
status_logs varchar (10) NOT NULL,
error_type_logs varchar (150),
client_logs varchar (100),
daemon_logs varchar (10) NOT NULL,
CONSTRAINT ind_logs PRIMARY KEY (id_logs)
);
CREATE INDEX logs_idx ON logs_table(id_logs);
......@@ -55,3 +55,21 @@ CREATE TABLE netidmap_table (
PRIMARY KEY (netid_netidmap, idp_netidmap, robot_netidmap)
);
CREATE INDEX netidmap_idx ON netidmap_table(netid_netidmap, idp_netidmap, robot_netidmap);
CREATE TABLE logs_table (
id_logs integer NOT NULL,