Commit 19a3fc0b authored by sikeda's avatar sikeda
Browse files

[dev] Refactoring: Now DKIM parameters are retrieved by bulk.pl: they need not...

[dev] Refactoring: Now DKIM parameters are retrieved by bulk.pl: they need not be stored into bulkspool table.
- $message->{shelved} attribute is set to indicate DKIM signing is required, instead of passing DKIM parameters.
- As a result, dkim_*_bulkspool columns are no longer used.

ToDo: Migration of {shelved}{dkim_sign} attributes during upgrade.


git-svn-id: https://subversion.renater.fr/sympa/branches/sympa-6.2-branch@11288 05aa8bb8-cd2b-0410-b1d7-8918dfa770ce
parent 604799be
......@@ -9255,10 +9255,9 @@ sub do_distribute {
unless (
$message
and defined Sympa::Mail::sending(
'message' => $message,
'rcpt' => Conf::get_robot_conf($robot, 'sympa'),
'from' => Conf::get_robot_conf($robot, 'request'),
'robot' => $robot,
$message,
Conf::get_robot_conf($robot, 'sympa'),
Conf::get_robot_conf($robot, 'request'),
'priority' => Conf::get_robot_conf($robot, 'sympa_priority'),
)
) {
......@@ -21123,17 +21122,16 @@ sub do_remind {
context => $robot);
$message->{envelope_sender} = Conf::get_robot_conf($robot, 'request');
$message->add_header('Message-ID', sprintf '<%s@wwsympa>', $time);
$message->add_header('From', $param->{'user'}{'email'});
$message->add_header('To', Conf::get_robot_conf($robot, 'sympa'));
$message->add_header('From', $param->{'user'}{'email'});
$message->add_header('To', Conf::get_robot_conf($robot, 'sympa'));
$message->add_header('Content-Type', 'text/plain; Charset=utf-8');
 
unless (
$message
and defined Sympa::Mail::sending(
'message' => $message,
'rcpt' => Conf::get_robot_conf($robot, 'sympa'),
'from' => Conf::get_robot_conf($robot, 'request'),
'robot' => $robot,
$message,
Conf::get_robot_conf($robot, 'sympa'),
Conf::get_robot_conf($robot, 'request'),
'priority' => Conf::get_robot_conf($robot, 'sympa_priority'),
)
) {
......@@ -22357,10 +22355,8 @@ sub do_send_mail {
unless (
$message
and defined Sympa::Mail::sending(
'message' => $message,
'rcpt' => $to,
'from' => Conf::get_robot_conf($robot, 'sympa'),
'robot' => $robot,
$message, $to,
Conf::get_robot_conf($robot, 'sympa'),
'priority' => Conf::get_robot_conf($robot, 'sympa_priority'),
)
) {
......@@ -22576,17 +22572,16 @@ sub do_tag_topic_by_sender {
context => $robot);
$message->{envelope_sender} = Conf::get_robot_conf($robot, 'request');
$message->add_header('Message-ID', sprintf '<%s@wwsympa>', $time);
$message->add_header('From', $sender);
$message->add_header('To', Conf::get_robot_conf($robot, 'sympa'));
$message->add_header('From', $sender);
$message->add_header('To', Conf::get_robot_conf($robot, 'sympa'));
$message->add_header('Content-Type', 'text/plain; Charset=utf-8');
 
unless (
$message
and defined Sympa::Mail::sending(
'message' => $message,
'rcpt' => Conf::get_robot_conf($robot, 'sympa'),
'from' => Conf::get_robot_conf($robot, 'request'),
'robot' => $robot,
$message,
Conf::get_robot_conf($robot, 'sympa'),
Conf::get_robot_conf($robot, 'request'),
'priority' => Conf::get_robot_conf($robot, 'sympa_priority'),
)
) {
......
......@@ -220,7 +220,7 @@ sub message_from_spool {
unless (
$sth = SDM::do_query(
"SELECT message_bulkspool AS message, messageid_bulkspool AS messageid, dkim_d_bulkspool AS dkim_d, dkim_i_bulkspool AS dkim_i, dkim_privatekey_bulkspool AS dkim_privatekey, dkim_selector_bulkspool AS dkim_selector FROM bulkspool_table WHERE messagekey_bulkspool = %s",
"SELECT message_bulkspool AS message, messageid_bulkspool AS messageid FROM bulkspool_table WHERE messagekey_bulkspool = %s",
SDM::quote($messagekey)
)
) {
......@@ -236,11 +236,7 @@ sub message_from_spool {
return (
{ 'messageasstring' =>
MIME::Base64::decode($message_from_spool->{'message'}),
'messageid' => $message_from_spool->{'messageid'},
'dkim_d' => $message_from_spool->{'dkim_d'},
'dkim_i' => $message_from_spool->{'dkim_i'},
'dkim_selector' => $message_from_spool->{'dkim_selector'},
'dkim_privatekey' => $message_from_spool->{'dkim_privatekey'},
'messageid' => $message_from_spool->{'messageid'},
}
);
......@@ -271,12 +267,11 @@ sub store {
$verp = 0 unless ($verp);
my $merge = $data{'merge'};
$merge = 0 unless ($merge);
my $dkim = $data{'dkim'};
my $tag_as_last = $data{'tag_as_last'};
Log::do_log(
'debug',
'(<msg>, <rcpts>, from = %s, robot = %s, listname= %s, priority_message = %s, delivery_date= %s, verp = %s, tracking = %s, merge = %s, dkim: d= %s i=%s, last: %s)',
'(<msg>, <rcpts>, from = %s, robot = %s, listname= %s, priority_message = %s, delivery_date= %s, verp = %s, tracking = %s, merge = %s, last: %s)',
$from,
$robot,
$listname,
......@@ -285,8 +280,6 @@ sub store {
$verp,
$tracking,
$merge,
$dkim->{'d'},
$dkim->{'i'},
$tag_as_last
);
......@@ -299,7 +292,7 @@ sub store {
my $parser = MIME::Parser->new();
$parser->output_to_core(1);
my $msg = $message->as_string;
my $msg = $message->to_string;
my $message_sender = $message->{'sender'};
$msg = MIME::Base64::encode($msg);
......@@ -345,33 +338,21 @@ sub store {
if ($message_already_on_spool == 0) {
my $statement = q{INSERT INTO bulkspool_table
(messagekey_bulkspool, messageid_bulkspool,
message_bulkspool, lock_bulkspool,
dkim_d_bulkspool, dkim_i_bulkspool,
dkim_selector_bulkspool, dkim_privatekey_bulkspool)
VALUES (?, ?, ?, 1, ?, ?, ?, ?)};
message_bulkspool, lock_bulkspool)
VALUES (?, ?, ?, 1)};
my $statementtrace = $statement;
$statementtrace =~ s/\n\s*/ /g;
$statementtrace =~ s/\?/\%s/g;
unless (
SDM::do_prepared_query(
$statement, $messagekey,
$msg_id, $msg,
$dkim->{d}, $dkim->{i},
$dkim->{selector}, $dkim->{private_key}
)
) {
unless (SDM::do_prepared_query(
$statement, $messagekey, $msg_id, $msg)) {
Log::do_log(
'err',
'Unable to add message in bulkspool_table "%s"',
sprintf($statementtrace,
SDM::quote($messagekey),
SDM::quote($msg_id),
SDM::quote(substr($msg, 0, 100)),
SDM::quote($dkim->{d}),
SDM::quote($dkim->{i}),
SDM::quote($dkim->{selector}),
SDM::quote(substr($dkim->{private_key}, 0, 30)))
SDM::quote(substr($msg, 0, 100)))
);
return undef;
}
......
......@@ -2712,18 +2712,13 @@ sub distribute {
if ( ($main::daemon_usage == Conf::DAEMON_MESSAGE)
|| ($main::daemon_usage == Conf::DAEMON_ALL)) {
my $numsmtp;
my $apply_dkim_signature = 'off';
$apply_dkim_signature = 'on'
if tools::is_in_array($list->{'admin'}{'dkim_signature_apply_on'},
'any');
$apply_dkim_signature = 'on'
$message->{shelved}{dkim_sign} = 1
if tools::is_in_array($list->{'admin'}{'dkim_signature_apply_on'},
'any')
or tools::is_in_array($list->{'admin'}{'dkim_signature_apply_on'},
'editor_validated_messages');
$numsmtp = $list->distribute_msg(
'message' => $message,
'apply_dkim_signature' => $apply_dkim_signature
);
$numsmtp = $list->distribute_msg($message);
unless (defined $numsmtp) {
Log::do_log('err', 'Unable to send message to list %s', $name);
Sympa::Report::reject_report_msg('intern', '', $sender,
......@@ -3031,20 +3026,14 @@ sub confirm {
if ( ($main::daemon_usage == Conf::DAEMON_MESSAGE)
|| ($main::daemon_usage == Conf::DAEMON_ALL)) {
my $numsmtp;
my $apply_dkim_signature = 'off';
$apply_dkim_signature = 'on'
if tools::is_in_array(
$list->{'admin'}{'dkim_signature_apply_on'}, 'any');
$apply_dkim_signature = 'on'
$message->{shelved}{dkim_sign} = 1
if tools::is_in_array(
$list->{'admin'}{'dkim_signature_apply_on'}, 'any')
or tools::is_in_array(
$list->{'admin'}{'dkim_signature_apply_on'},
'md5_authenticated_messages');
$numsmtp = $list->distribute_msg(
'message' => $message,
'apply_dkim_signature' => $apply_dkim_signature
);
$numsmtp = $list->distribute_msg($message);
unless (defined $numsmtp) {
Log::do_log('err', 'Unable to send message to list %s',
$list->{'name'});
......
......@@ -232,30 +232,30 @@ my %full_db_struct = (
'when set to 1, this field prevents Sympa from processing the message',
'order' => 5,
},
'dkim_privatekey_bulkspool' => {
'struct' => 'varchar(2000)',
'doc' =>
'DKIM parameter stored for bulk daemon because bulk ignore list parameters, private key to sign message',
'order' => 6,
},
'dkim_selector_bulkspool' => {
'struct' => 'varchar(50)',
'doc' =>
'DKIM parameter stored for bulk daemon because bulk ignore list parameters, DKIM selector to sign message',
'order' => 7,
},
'dkim_d_bulkspool' => {
'struct' => 'varchar(50)',
'doc' =>
'DKIM parameter stored for bulk daemon because bulk ignore list parameters, the d DKIM parameter',
'order' => 8,
},
'dkim_i_bulkspool' => {
'struct' => 'varchar(100)',
'doc' =>
'DKIM parameter stored for bulk daemon because bulk ignore list parameters, DKIM i signature parameter',
'order' => 9,
},
#'dkim_privatekey_bulkspool' => {
# 'struct' => 'varchar(2000)',
# 'doc' =>
# 'DKIM parameter stored for bulk daemon because bulk ignore list parameters, private key to sign message',
# 'order' => 6,
#},
#'dkim_selector_bulkspool' => {
# 'struct' => 'varchar(50)',
# 'doc' =>
# 'DKIM parameter stored for bulk daemon because bulk ignore list parameters, DKIM selector to sign message',
# 'order' => 7,
#},
#'dkim_d_bulkspool' => {
# 'struct' => 'varchar(50)',
# 'doc' =>
# 'DKIM parameter stored for bulk daemon because bulk ignore list parameters, the d DKIM parameter',
# 'order' => 8,
#},
#'dkim_i_bulkspool' => {
# 'struct' => 'varchar(100)',
# 'doc' =>
# 'DKIM parameter stored for bulk daemon because bulk ignore list parameters, DKIM i signature parameter',
# 'order' => 9,
#},
},
'doc' => 'This table contains the messages to be sent by bulk.pl',
'order' => 3,
......
......@@ -1562,18 +1562,12 @@ sub _get_single_param_value {
#
# IN : -$self (+): ref(List)
# -$message (+): ref(Message)
# -$apply_dkim_signature : on | off
# OUT : -$numsmtp : number of sendmail process
####################################################
sub distribute_msg {
my $self = shift;
my %param = @_;
my $message = $param{'message'};
my $apply_dkim_signature = $param{'apply_dkim_signature'};
Log::do_log('debug2', '(%s, message=%s, apply_dkim_signature=%s)',
$self, $message, $apply_dkim_signature);
Log::do_log('debug2', '(%s, %s)', @_);
my $self = shift;
my $message = shift;
my ($name, $host) = ($self->{'name'}, $self->{'admin'}{'host'});
my $robot = $self->{'domain'};
......@@ -2052,11 +2046,8 @@ sub distribute_msg {
}
## Blindly send the message to all users.
my $numsmtp = $self->send_msg(
$message,
'apply_dkim_signature' => $apply_dkim_signature,
'apply_tracking' => $apply_tracking
);
my $numsmtp =
$self->send_msg($message, 'apply_tracking' => $apply_tracking);
$self->savestats() if (defined($numsmtp));
return $numsmtp;
......@@ -2604,34 +2595,26 @@ sub send_file {
$data->{'boundary'} = '----------=_' . tools::get_message_id($robot)
unless ($data->{'boundary'});
$data->{'sign_mode'} = $sign_mode;
if ((Conf::get_robot_conf($self->{'domain'}, 'dkim_feature') eq 'on')
&& (Conf::get_robot_conf($self->{'domain'}, 'dkim_add_signature_to')
=~ /robot/)
) {
$data->{'dkim'} =
tools::get_dkim_parameters({'robot' => $self->{'domain'}});
}
my $message =
Sympa::Message->new_from_template($self, $filename, $who, $data);
$message->{shelved}{dkim_sign} = 1
if Conf::get_robot_conf($self->{'domain'}, 'dkim_feature') eq 'on'
and
Conf::get_robot_conf($self->{'domain'}, 'dkim_add_signature_to')
=~ /robot/;
# use verp excepted for alarms. We should make this configurable in order
# to support Sympa server on a machine without any MTA service
$data->{'use_bulk'} = 1
unless ($data->{'alarm'});
my $use_bulk = 1 unless $data->{'alarm'};
my $message =
Sympa::Message->new_from_template($self, $filename, $who, $data);
unless (
$message
and defined Sympa::Mail::sending(
'message' => $message,
'rcpt' => $who,
'from' => $data->{'return_path'},
'robot' => $robot,
'listname' => $self->{'name'},
$message, $who, $data->{'return_path'},
'priority' => Conf::get_robot_conf($robot, 'sympa_priority'),
'sign_mode' => $data->{'sign_mode'},
'use_bulk' => $data->{'use_bulk'},
'dkim' => $data->{'dkim'},
'sign_mode' => $sign_mode,
'use_bulk' => $use_bulk,
)
) {
Log::do_log('err', 'Could not send template %s to %s',
......@@ -2653,7 +2636,7 @@ sub send_file {
#
#
# IN : -$self (+): ref(List)
# -$message (+): ref(Message)
# -$message (+): ref(Message), possiblly {shelved}{dkim_sign} flag set.
# OUT : -$numsmtp : number of sendmail process
# | 0 : no subscriber for sending message in list
# | undef
......@@ -2664,8 +2647,7 @@ sub send_msg {
my $message = shift;
my %param = @_;
my $apply_dkim_signature = $param{'apply_dkim_signature'};
my $apply_tracking = $param{'apply_tracking'};
my $apply_tracking = $param{'apply_tracking'};
my $original_message_id = $message->{'message_id'};
my $robot = $self->{'domain'};
......@@ -2717,13 +2699,6 @@ sub send_msg {
$tags_to_use->{'tag_noverp'} = 0;
}
my $dkim_parameters;
# prepare dkim parameters
if ($apply_dkim_signature eq 'on') {
$dkim_parameters = tools::get_dkim_parameters(
{'robot' => $self->{'domain'}, 'listname' => $self->{'name'}});
}
# Separate subscribers depending on user reception option and also if verp
# a dicovered some bounce for them.
# Storing the not empty subscribers' arrays into a hash.
......@@ -2769,12 +2744,11 @@ sub send_msg {
if ($#selected_tabrcpt > -1) {
my $result = Sympa::Mail::mail_message(
'message' => $new_message,
'rcpt' => \@selected_tabrcpt,
'list' => $self,
'verp' => 'off',
'dkim_parameters' => $dkim_parameters,
'tag_as_last' => $tags_to_use->{'tag_noverp'}
'message' => $new_message,
'rcpt' => \@selected_tabrcpt,
'list' => $self,
'verp' => 'off',
'tag_as_last' => $tags_to_use->{'tag_noverp'}
);
unless (defined $result) {
Log::do_log(
......@@ -2821,12 +2795,11 @@ sub send_msg {
## prepare VERP sending.
if (@verp_selected_tabrcpt) {
my $result = Sympa::Mail::mail_message(
'message' => $new_message,
'rcpt' => \@verp_selected_tabrcpt,
'list' => $self,
'verp' => 'on',
'dkim_parameters' => $dkim_parameters,
'tag_as_last' => $tags_to_use->{'tag_verp'}
'message' => $new_message,
'rcpt' => \@verp_selected_tabrcpt,
'list' => $self,
'verp' => 'on',
'tag_as_last' => $tags_to_use->{'tag_verp'}
);
unless (defined $result) {
Log::do_log(
......
......@@ -104,7 +104,6 @@ sub mail_message {
my $list = $params{'list'};
my $verp = $params{'verp'};
my @rcpt = @{$params{'rcpt'}};
my $dkim = $params{'dkim_parameters'};
my $tag_as_last = $params{'tag_as_last'};
my $host = $list->{'admin'}{'host'};
......@@ -235,7 +234,6 @@ sub mail_message {
'encrypt' => $message->{'smime_crypted'},
'use_bulk' => 1,
'verp' => $verp,
'dkim' => $dkim,
'merge' => $list->{'admin'}{'merge_feature'},
'tag_as_last' => $tag_as_last
)
......@@ -269,10 +267,7 @@ sub mail_forward {
unless (
defined sending(
'message' => $message,
'rcpt' => $rcpt,
'from' => $from,
'robot' => $robot,
$message, $rcpt, $from,
'priority' => Conf::get_robot_conf($robot, 'request_priority'),
)
) {
......@@ -350,7 +345,6 @@ sub sendto {
my $encrypt = $params{'encrypt'};
my $verp = $params{'verp'};
my $merge = $params{'merge'};
my $dkim = $params{'dkim'};
my $use_bulk = $params{'use_bulk'};
my $tag_as_last = $params{'tag_as_last'};
......@@ -399,11 +393,7 @@ sub sendto {
}
unless (
sending(
'message' => $new_message,
'rcpt' => $email,
'from' => $from,
'listname' => $listname,
'robot' => $robot,
$new_message, $email, $from,
'priority' => $priority,
'delivery_date' => $delivery_date,
'use_bulk' => $use_bulk,
......@@ -418,17 +408,12 @@ sub sendto {
}
} else {
my $result = sending(
'message' => $message,
'rcpt' => $rcpt,
'from' => $from,
'listname' => $listname,
'robot' => $robot,
$message, $rcpt, $from,
'priority' => $priority,
'delivery_date' => $delivery_date,
'verp' => $verp,
'merge' => $merge,
'use_bulk' => $use_bulk,
'dkim' => $dkim,
'tag_as_last' => $tag_as_last
);
return $result;
......@@ -444,39 +429,46 @@ sub sendto {
# Signing if needed
#
#
# IN : -$msg(+) : ref(MIME::Entity) | string - message to send
# -$rcpt(+) : ref(SCALAR) | ref(ARRAY) - recepients
# (for SMTP : "RCPT To:" field)
# -$from(+) : for SMTP "MAIL From:" field , for
# spool sending : "X-Sympa-From" field
# -$robot(+) : robot
# -$listname : listname | ''
# -$sign_mode(+) : 'smime' | 'none' for signing
# -$verp
# -dkim : a hash for dkim parameters
# IN : -$message: ref(Sympa::Message) - message to be sent
# -$rcpt: SCALAR | ref(ARRAY) - recipients for SMTP "RCPT To:" field.
# -$from: for SMTP, "MAIL From:" field; for spool sending, "X-Sympa-From"
# field
# -sign_mode => mode: 'smime' for signing
# -verp => 'on' | 'mdn' | 'dsn' | undef
# -merge => 'on' | 'off'
# -use_bulk => boolean
#
# OUT : 1 - call to smtpto (sendmail) | 0 - push in spool
# | undef
#
####################################################
sub sending {
my %params = @_;
my $message = $params{'message'};
my $rcpt = $params{'rcpt'};
my $from = $params{'from'};
my $robot = $params{'robot'};
my $listname = $params{'listname'};
my $message = shift;
my $rcpt = shift;
my $from = shift;
my %params = @_;
my $that = $message->{context};
my ($robot_id, $listname);
if (ref $that eq 'Sympa::List') {
$robot_id = $that->{'domain'};
$listname = $that->{'name'};
} elsif ($that and $that ne '*') {
$robot_id = $that;
} else {
$robot_id = '*';
}
my $sign_mode = $params{'sign_mode'};
my $sympa_email = $params{'sympa_email'};
my $priority_message = $params{'priority'};
my $priority_packet =
Conf::get_robot_conf($robot, 'sympa_packet_priority');
Conf::get_robot_conf($robot_id, 'sympa_packet_priority');
my $delivery_date = $params{'delivery_date'};
$delivery_date = time() unless ($delivery_date);
my $verp = $params{'verp'};
my $merge = $params{'merge'};
my $use_bulk = $params{'use_bulk'};
my $dkim = $params{'dkim'};
my $tag_as_last = $params{'tag_as_last'};
my $sympa_file;
my $fh;
......@@ -505,7 +497,7 @@ sub sending {
'message' => $message,
'rcpts' => $rcpt,
'from' => $from,
'robot' => $robot,
'robot' => $robot_id,
'listname' => $listname,
'priority_message' => $priority_message,
'priority_packet' => $priority_packet,
......@@ -513,14 +505,13 @@ sub sending {
'verp' => $verpfeature,
'tracking' => $trackingfeature,
'merge' => $mergefeature,
'dkim' => $dkim,
'tag_as_last' => $tag_as_last,
);
unless (defined $bulk_code) {
Log::do_log('err', 'Failed to store message for list %s',
$listname);
Sympa::Robot::send_notify_to_listmaster('bulk_error', $robot,
Sympa::Robot::send_notify_to_listmaster('bulk_error', $robot_id,
{'listname' => $listname});
return undef;
}
......@@ -529,7 +520,7 @@ sub sending {
# it to standard spool
Log::do_log('debug', "NOT USING BULK");
$sympa_email = Conf::get_robot_conf($robot, 'sympa');
$sympa_email = Conf::get_robot_conf($robot_id, 'sympa');
$sympa_file =
"$send_spool/T.$sympa_email." . time . '.' . int(rand(10000));
unless (open TMP, ">$sympa_file") {
......@@ -562,7 +553,7 @@ sub sending {
}
} else { # send it now
Log::do_log('debug', "NOT USING BULK");
*SMTP = smtpto($from, $rcpt, $robot);
*SMTP = smtpto($from, $rcpt, $robot_id);
# Send message stripping Return-Path pseudo-header field.
my $msg_string = $message->as_string;
......
......@@ -156,6 +156,8 @@ sub new {
$self->{'envelope_sender'} = $v;
} elsif ($k eq 'X-Sympa-Spam-Status') { # New in 6.2a.41
$self->{'spam_status'} = $v;
} elsif ($k eq 'X-Sympa-Shelved') { # New in 6.2a.41
$self->{'shelved'} = {map { ($_ => 1) } split /\s*,\s*/, $v};
} else {
Log::do_log('err', 'Unknown meta information: "%s: %s"', $k, $v);
}
......@@ -715,6 +717,12 @@ sub to_string {
$serialized .= sprintf "X-Sympa-Spam-Status: %s\n",
$self->{'spam_status'};
}
if (defined $self->{'shelved'} and %{$self->{'shelved'}}) {
$serialized .= sprintf "X-Sympa-Shelved: %s\n",
join(',',
grep { $self->{shelved}{$_} }
sort keys %{$self->{shelved}});
}
# This terminates pseudo-header part for attributes.
unless (defined $self->{'envelope_sender'}) {
$serialized .= "Return-Path: \n";
......@@ -3091,13 +3099,26 @@ Get unique identifier of instance.
sub get_id {
my $self = shift;
my $id;
# Tentative. Alternatives for more general ID in the future.
return $self->{'messagekey'} if $self->{'messagekey'};
if ($self->{'filename'}) {
if ($self->{'messagekey'}) {
$id = $self->{'messagekey'};
} elsif ($self->{'filename'}) {
my @parts = split /\//, $self->{'filename'};
return pop @parts;
$id = pop @parts;
} else {
$id = $self->{'message_id'};
}