Commit 233415d7 authored by david.verdin's avatar david.verdin
Browse files

[feature][Submitted by J. jourdan] Messages parsing; custom attributes can be...

[feature][Submitted by J. jourdan] Messages parsing; custom attributes can be now used as parameters for TT2 tags in messages distributed by Sympa. This adds notably the possibility to add authenticated unsubscribe URL to message footers.


git-svn-id: https://subversion.renater.fr/sympa/trunk@5998 05aa8bb8-cd2b-0410-b1d7-8918dfa770ce
parent a7276755
......@@ -39,6 +39,8 @@ This script must be run along with sympa. It regularly checks the bulkmailer_tab
use lib '--modulesdir--';
use strict;
use Conf;
use Log;
use Commands;
......@@ -154,14 +156,16 @@ my $opensmtp = 0 ;
my $fh = 'fh0000000000'; ## File handle for the stream.
my $messagekey; # the key of the current message in the message_table
my $messageasstring; # the current message as a string
my $messageasstring_init; # the current message as a string
my $timeout = $Conf::Conf{'bulk_wait_to_fork'};
my $last_check_date = time();
$options->{'multiple_process'} = 1;
$Conf::Conf{'maxsmtp'} = int($Conf::Conf{'maxsmtp'}/$Conf::Conf{'bulk_max_count'});
while (!$end) {
&List::init_list_cache();
my $bulk;
unless ($main::options{'foreground'}) {
......@@ -212,49 +216,109 @@ while (!$end) {
## Go through the bulk_mailer table and process messages
if ($bulk = Bulk::next()) {
if ($bulk->{'messagekey'} ne $messagekey) {
# current packet is no related to the same message as the previous packet
if ($bulk->{'messagekey'} ne $messagekey) {
# current packet is no related to the same message as the previous packet
# so it is needed to fetch the new message from message_table
$messageasstring = &Bulk::messageasstring($bulk->{'messagekey'});
unless ( $messageasstring ) {
&do_log('err',"internal error : current packet 'messagekey= %s contain a ref to a null message",$bulk->{'messagekey'});
}
}
my @rcpts = split /,/,$bulk->{'receipients'};
if ($bulk->{'verp'}){
foreach my $rcpt (@rcpts) {
$return_path = $rcpt;
$return_path =~ s/\@/\=\=a\=\=/;
$return_path = "$Conf::Conf{'bounce_email_prefix'}+$return_path\=\=$bulk->{'listname'}\@$bulk->{'robot'}"; # VERP broken if no listname is provided
*SMTP = &mail::smtpto($return_path, \$rcpt, $bulk->{'robot'});
print SMTP $messageasstring;
close SMTP;
}
$messageasstring_init = &Bulk::messageasstring($bulk->{'messagekey'});
unless ( $messageasstring_init ) {
&do_log('err',"internal error : current packet 'messagekey= %s contain a ref to a null message",$bulk->{'messagekey'});
}
}
#--------------------------------------------------
#------------- BEGIN VERP AND MERGE ---------------
#--------------------------------------------------
my $data; #HASH which will contain the attributes of the subscriber
# Initialization of the HASH : $data. It will be used by parse_tt2 to personalized messages.
$data->{'listname'} = $bulk->{'listname'};
$data->{'robot'} = $bulk->{'robot'};
$data->{'to'} = $bulk->{'receipients'};
$data->{'wwsympa_url'} = $Conf{'wwsympa_url'};
my $rcpt; # It is the email of a subscriber, use it in the foreach
my @rcpts = split /,/,$bulk->{'receipients'}; # Contain all the subscribers
## Use an intermediate handler to encode to filesystem_encoding
my $user;
#____ Test if use verp ______
if ($bulk->{'verp'}){
foreach $rcpt (@rcpts) {
$return_path = $rcpt;
$return_path =~ s/\@/\=\=a\=\=/;
$return_path = "$Conf::Conf{'bounce_email_prefix'}+$return_path\=\=$bulk->{'listname'}\@$bulk->{'robot'}"; # xxxxxxxxxxxxx verp cassé si pas de listename (message de sympa)
#____ Test if use merge ______
if (1==1) { #-------- it will be : if ($bulk->{'merge'}) { ------------#
### Parse an MIME::Entity message
my $parser = new MIME::Parser;
my $entity = $parser->parse_data($messageasstring_init); ## Retrouver la méthode qui correspond
unless(&Bulk::merge_msg ($entity, $rcpt, $bulk, $data)){
&do_log('err', 'Erreur d appel &Bulk::merge_msg');
}
}else{
*SMTP = &mail::smtpto($bulk->{'returnpath'}, \@rcpts, $bulk->{'robot'});
print SMTP $messageasstring;
close SMTP;
my $messageasstring = $entity->as_string;
}
*SMTP = &mail::smtpto($return_path, \$rcpt, $bulk->{'robot'});
# Message with customized data
print SMTP $messageasstring;
close SMTP;
}
}else{
## Remove packet once it has been processed
unless (&Bulk::remove($bulk->{'messagekey'},$bulk->{'packetid'})) {
&do_log('err',"failed to remove processed packet '%s', messagekey '%s'", $bulk->{'messagekey'},$bulk->{'packetid'});
return undef;
}
#____ Test if use merge ______
if ( 1==1 ) { #-------- it will be : if ($bulk->{'merge'}) { ------------#
foreach $rcpt (@rcpts) {
### Parse an MIME::Entity message
my $parser = new MIME::Parser;
my $entity = $parser->parse_data($messageasstring_init);
if($bulk->{'priority_packet'} == $Conf::Conf{'sympa_packet_priority'} + 1){
&do_log('notice','Done sending message %s to list %s@%s (priority %s) in %s seconds since scheduled expedition date. Now sending VERP.',
$bulk->{'messagekey'},
$bulk->{'listname'},
$bulk->{'robot'},
$bulk->{'priority_message'},
time() - $bulk->{'delivery_date'});
unless(&Bulk::merge_msg ($entity, $rcpt, $bulk, $data)){
&do_log('err', 'Erreur d appel &Bulk::merge_msg');
}
my $messageasstring = $entity->as_string;
*SMTP = &mail::smtpto($bulk->{'returnpath'}, \$rcpt, $bulk->{'robot'});
# Message with customized data
print SMTP $messageasstring;
close SMTP;
}
$date_of_last_activity = time();
}else{
*SMTP = &mail::smtpto($bulk->{'returnpath'}, \@rcpts, $bulk->{'robot'});
# Initial message
print SMTP $messageasstring_init;
close SMTP;
}
}
#--------------------------------------------------
#------------- END VERP AND MERGE -----------------
#--------------------------------------------------
## Remove packet once it has been processed
unless (&Bulk::remove($bulk->{'messagekey'},$bulk->{'packetid'})) {
&do_log('err',"failed to remove processed packet '%s', messagekey '%s'", $bulk->{'messagekey'},$bulk->{'packetid'});
return undef;
}
if($bulk->{'priority_packet'} == $Conf::Conf{'sympa_packet_priority'} + 1){
&do_log('notice','Done sending message %s to list %s@%s (priority %s) in %s seconds since scheduled expedition date. Now sending VERP.',
$bulk->{'messagekey'},
$bulk->{'listname'},
$bulk->{'robot'},
$bulk->{'priority_message'},
time() - $bulk->{'delivery_date'});
}
$date_of_last_activity = time();
}else{
## Sleep for a while if bulk_mailer DB table is empty
sleep $Conf::Conf{'bulk_sleep'};
## Sleep for a while if bulk_mailer DB table is empty
sleep $Conf::Conf{'bulk_sleep'};
}
&mail::reaper;
}
......
......@@ -154,6 +154,159 @@ sub messageasstring {
return( MIME::Base64::decode($messageasstring->{'message'}) );
}
############################################################
# merge_msg #
############################################################
# #
# #
# #
# #
# #
# #
############################################################
sub merge_msg {
my $entity = shift;
my $rcpt = shift;
my $bulk = shift;
my $data = shift;
## Test MIME::Entity
unless (defined $entity && ref($entity) eq 'MIME::Entity') {
&do_log('err', 'echec entity');
return undef;
}
my $body;
if(defined $entity->bodyhandle){
$body = $entity->bodyhandle->as_string;
}
## Get the Content-Type / Charset / Content-Transfer-encoding of a message
my $type = $entity->mime_type;
my $charset = unmime $entity->head->mime_attr('content-type.charset');
my $encoding = unmime $entity->head->mime_encoding;
my $message_output;
my $IO;
## If Content-Type is a text/*
if($entity->mime_type =~ /^text/){
if(defined $body){
## --------- Initial Charset to UTF-8 --------- ##
unless($charset =~ /UTF-8/){
# Put the charset to UTF-8
Encode::from_to($body, $charset, 'UTF-8');
}
## PARSAGE ##
&merge_data('rcpt' => $rcpt,
'listname' => $bulk->{'listname'},
'robot' => $bulk->{'robot'},
'data' => $data,
'body' => $body,
'message_output' => \$message_output,
);
$body = $message_output;
unless($charset =~ /UTF-8/){
# Put the charset to the initial
Encode::from_to($body, 'UTF-8',$charset);
}
# Write the new body in the entity
unless($IO = $entity->bodyhandle->open("w") || die "open body: $!"){
&do_log('err', "Can't open Entity");
return undef;
}
unless($IO->print($body)){
&do_log('err', "Can't write in Entity");
return undef;
}
unless($IO->close || die "close I/O handle: $!"){
&do_log('err', "Can't close Entity");
return undef;
}
}
}
##--- Recursive call of the method. ---##
## Course on the different parts of the message at all levels.
foreach my $part ($entity->parts) {
unless(&merge_msg($part, $rcpt, $bulk, $data)){
&do_log('err', "Echec d'appel &merge_msg");
return undef;
}
}
return 1;
}
############################################################
# merge_data #
############################################################
# This function retrieves the customized data of the #
# users then parse the message. It returns the message #
# personalized to bulk.pl #
# It uses the method &tt2::parse_tt2 #
# It uses the method &List::get_subscriber_no_object #
# It uses the method &tools::get_fingerprint #
# #
# IN : - rcpt : the receipient email #
# - listname : the name of the list #
# - robot : the host #
# - data : HASH with many data #
# - body : message with the TT2 #
# - message_output : object, IO::Scalar #
# #
# OUT : - message_output : customized message #
# | undef #
# #
############################################################
sub merge_data {
my %params = @_;
my $rcpt = $params{'rcpt'},
my $listname = $params{'listname'},
my $robot = $params{'robot'},
my $data = $params{'data'},
my $body = $params{'body'},
my $message_output = $params{'message_output'},
my $options;
$options->{'is_not_template'} = 1;
my $user_details;
$user_details->{'email'} = $rcpt;
$user_details->{'name'} = $listname;
$user_details->{'domain'} = $robot;
# get_subscriber_no_object() return the user's details with the custom attributes
my $user = &List::get_subscriber_no_object($user_details);
$user->{'friendly_date'} = &POSIX::strftime("%d %b %Y %H:%M", localtime($user->{'date'}));
$user->{'fingerprint'} = &tools::get_fingerprint($rcpt);
my $url = $data->{'wwsympa_url'};
$data = {
'user' => $user,
'robot' => $robot,
'listname' => $listname,
'url' => $url,
};
# Parse the TT2 in the message : replace the tags and the parameters by the corresponding values
unless (&tt2::parse_tt2($data,\$body, $message_output, '', $options)) {
&do_log('err','Unable to parse body : "%s"', \$body);
return undef;
}
return 1;
}
##
sub store {
my %data = @_;
......
......@@ -3147,57 +3147,65 @@ sub send_msg {
my $mixed = ($message->{'msg'}->head->get('Content-Type') =~ /multipart\/mixed/i);
my $alternative = ($message->{'msg'}->head->get('Content-Type') =~ /multipart\/alternative/i);
for ( my $user = $self->get_first_user(); $user; $user = $self->get_next_user() ){
unless ($user->{'email'}) {
&do_log('err','Skipping user with no email address in list %s', $name);
next;
}
if ($user->{'reception'} =~ /^digest|digestplain|summary|nomail$/i) {
next;
} elsif ($user->{'reception'} eq 'notice') {
if ($user->{'bounce_address'}) {
push @tabrcpt_notice_verp, $user->{'email'};
}else{
push @tabrcpt_notice, $user->{'email'};
}
} elsif ($alternative and ($user->{'reception'} eq 'txt')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_txt_verp, $user->{'email'};
}else{
push @tabrcpt_txt, $user->{'email'};
if ( $message->{'msg'}->head->get('X-Sympa-Receipient') ) {
@tabrcpt = split /,/, $message->{'msg'}->head->get('X-Sympa-Receipient');
$message->{'msg'}->head->delete('X-Sympa-Receipient');
} else {
for ( my $user = $self->get_first_user(); $user; $user = $self->get_next_user() ){
unless ($user->{'email'}) {
&do_log('err','Skipping user with no email address in list %s', $name);
next;
}
} elsif ($alternative and ($user->{'reception'} eq 'html')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_html_verp, $user->{'email'};
}else{
if ($user->{'reception'} =~ /^digest|digestplain|summary|nomail$/i) {
next;
} elsif ($user->{'reception'} eq 'notice') {
if ($user->{'bounce_address'}) {
push @tabrcpt_notice_verp, $user->{'email'};
}else{
push @tabrcpt_notice, $user->{'email'};
}
} elsif ($alternative and ($user->{'reception'} eq 'txt')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_txt_verp, $user->{'email'};
}else{
push @tabrcpt_txt, $user->{'email'};
}
} elsif ($alternative and ($user->{'reception'} eq 'html')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_html_verp, $user->{'email'};
}else{
push @tabrcpt_html, $user->{'email'};
if ($user->{'bounce_address'}) {
push @tabrcpt_html_verp, $user->{'email'};
}else{
push @tabrcpt_html, $user->{'email'};
}
}
} elsif ($mixed and ($user->{'reception'} eq 'urlize')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_url_verp, $user->{'email'};
}else{
push @tabrcpt_url, $user->{'email'};
}
} elsif ($message->{'smime_crypted'} &&
(! -r $Conf::Conf{'ssl_cert_dir'}.'/'.&tools::escape_chars($user->{'email'}) &&
! -r $Conf::Conf{'ssl_cert_dir'}.'/'.&tools::escape_chars($user->{'email'}.'@enc' ))) {
## Missing User certificate
unless ($self->send_file('x509-user-cert-missing', $user->{'email'}, $robot, {'mail' => {'subject' => $message->{'msg'}->head->get('Subject'),
'sender' => $message->{'msg'}->head->get('From')},
'auto_submitted' => 'auto-generated'})) {
&do_log('notice',"Unable to send template 'x509-user-cert-missing' to $user->{'email'}");
}
}
} elsif ($mixed and ($user->{'reception'} eq 'urlize')) {
if ($user->{'bounce_address'}) {
push @tabrcpt_url_verp, $user->{'email'};
}else{
push @tabrcpt_url, $user->{'email'};
}
} elsif ($message->{'smime_crypted'} &&
(! -r $Conf::Conf{'ssl_cert_dir'}.'/'.&tools::escape_chars($user->{'email'}) &&
! -r $Conf::Conf{'ssl_cert_dir'}.'/'.&tools::escape_chars($user->{'email'}.'@enc' ))) {
## Missing User certificate
unless ($self->send_file('x509-user-cert-missing', $user->{'email'}, $robot, {'mail' => {'subject' => $message->{'msg'}->head->get('Subject'),
'sender' => $message->{'msg'}->head->get('From')},
'auto_submitted' => 'auto-generated'})) {
&do_log('notice',"Unable to send template 'x509-user-cert-missing' to $user->{'email'}");
}
}else{
if ($user->{'bounce_address'}) {
push @tabrcpt_verp, $user->{'email'} unless ($sender_hash{$user->{'email'}})&&($user->{'reception'} eq 'not_me');
}else{
push @tabrcpt, $user->{'email'} unless ($sender_hash{$user->{'email'}})&&($user->{'reception'} eq 'not_me');}
if ($user->{'bounce_address'}) {
push @tabrcpt_verp, $user->{'email'} unless ($sender_hash{$user->{'email'}})&&($user->{'reception'} eq 'not_me');
}else{
push @tabrcpt, $user->{'email'} unless ($sender_hash{$user->{'email'}})&&($user->{'reception'} eq 'not_me');}
}
}
}
}
## sa return 0 = Pb ?
unless (@tabrcpt || @tabrcpt_notice || @tabrcpt_txt || @tabrcpt_html || @tabrcpt_url || @tabrcpt_verp || @tabrcpt_notice_verp || @tabrcpt_txt_verp || @tabrcpt_html_verp || @tabrcpt_url_verp) {
......@@ -4762,75 +4770,116 @@ sub get_subscriber {
my $update_field = sprintf $date_format{'read'}{$Conf::Conf{'db_type'}}, 'update_subscriber', 'update_subscriber';
## Use session cache
if (defined $list_cache{'get_subscriber'}{$self->{'domain'}}{$name}{$email}) {
return $list_cache{'get_subscriber'}{$self->{'domain'}}{$name}{$email};
if (defined $list_cache{'get_subscriber'}{$self->{'domain'}}{$self->{'name'}}{$email}) {
return $list_cache{'get_subscriber'}{$self->{'domain'}}{$self->{'name'}}{$email};
}
my $options;
$options->{'email'} = $email;
$options->{'name'} = $self->{'name'};
$options->{'domain'} = $self->{'domain'};
my $user = &get_subscriber_no_object($options);
unless($user){
do_log('err','Unable to retrieve information from database for user %s', $email);
return undef;
}
$user->{'reception'} = $self->{'admin'}{'default_user_options'}{'reception'}
unless ($self->is_available_reception_mode($user->{'reception'}));
## In case it was not set in the database
$user->{'subscribed'} = 1 if ($self->{'admin'}{'user_data_source'} eq 'database');
## Set session cache
$list_cache{'get_subscriber'}{$self->{'domain'}}{$self->{'name'}}{$email} = $user;
return $user;
}
######################################################################
### get_subscriber_no_object #
## Get details regarding a subscriber. #
# IN: #
# - a single reference to a hash with the following keys: #
# * email : the subscriber email #
# * listname: the name of the list #
# * domain: the virtual host under which the list is installed. #
# OUT: #
# - undef if something went wrong. #
# - a hash containing the user details otherwise #
######################################################################
sub get_subscriber_no_object {
my $options = shift;
&do_log('debug2', 'List::get_subscriber_no_object(%s, %s, %s)', $options->{'name'}, $options->{'email'}, $options->{'domain'});
my $name = $options->{'name'};
my $email = &tools::clean_email($options->{'email'});
my $statement;
my $date_field = sprintf $date_format{'read'}{$Conf::Conf{'db_type'}}, 'date_subscriber', 'date_subscriber';
my $update_field = sprintf $date_format{'read'}{$Conf::Conf{'db_type'}}, 'update_subscriber', 'update_subscriber';
## Use session cache
if (defined $list_cache{'get_subscriber'}{$options->{'domain'}}{$name}{$email}) {
return $list_cache{'get_subscriber'}{$options->{'domain'}}{$name}{$email};
}
## Check database connection
unless ($dbh and $dbh->ping) {
return undef unless &db_connect();
}
## Additional subscriber fields
my $additional;
if ($Conf::Conf{'db_additional_subscriber_fields'}) {
$additional = ',' . $Conf::Conf{'db_additional_subscriber_fields'};
}
$statement = sprintf "SELECT user_subscriber AS email, comment_subscriber AS gecos, bounce_subscriber AS bounce, bounce_score_subscriber AS bounce_score, bounce_address_subscriber AS bounce_address, reception_subscriber AS reception, topics_subscriber AS topics, visibility_subscriber AS visibility, %s AS date, %s AS update_date, subscribed_subscriber AS subscribed, included_subscriber AS included, include_sources_subscriber AS id, custom_attribute_subscriber AS custom_attribute %s FROM subscriber_table WHERE (user_subscriber = %s AND list_subscriber = %s AND robot_subscriber = %s)",
$date_field,
$update_field,
$additional,
$dbh->quote($email),
$dbh->quote($name),
$dbh->quote($self->{'domain'});
$date_field,
$update_field,
$additional,
$dbh->quote($email),
$dbh->quote($name),
$dbh->quote($options->{'domain'});
push @sth_stack, $sth;
unless ($sth = $dbh->prepare($statement)) {
do_log('err','Unable to prepare SQL statement : %s', $dbh->errstr);
return undef;
}
unless ($sth->execute) {
do_log('err','Unable to execute SQL statement "%s" : %s', $statement, $dbh->errstr);
return undef;
}
my $user = $sth->fetchrow_hashref('NAME_lc');
if (defined $user) {
$user->{'reception'} ||= 'mail';
$user->{'reception'} = $self->{'admin'}{'default_user_options'}{'reception'}
unless ($self->is_available_reception_mode($user->{'reception'}));
$user->{'reception'} ||= 'mail';
$user->{'update_date'} ||= $user->{'date'};
## In case it was not set in the database
$user->{'subscribed'} = 1 if ($self->{'admin'}{'user_data_source'} eq 'database');
do_log('debug2', 'List::get_subscriber custom_attribute = (%s)', $user->{custom_attribute});
do_log('debug2', 'custom_attribute = (%s)', $user->{custom_attribute});
if (defined $user->{custom_attribute}) {
do_log('debug2', '1. custom_attribute = (%s)', $user->{custom_attribute});
my %custom_attr = &parseCustomAttribute($user->{'custom_attribute'});
$user->{'custom_attribute'} = \%custom_attr ;
do_log('debug2', '2. custom_attribute = (%s)', %custom_attr);
do_log('debug2', '3. custom_attribute = (%s)', $user->{custom_attribute});
do_log('debug2', '3. custom_attribute = (%s)', $user->{custom_attribute});
my @k = sort keys %custom_attr ;
do_log('debug2', "keys custom_attribute = @k");
}
}
$sth->finish();
$sth = pop @sth_stack;
## Set session cache
$list_cache{'get_subscriber'}{$self->{'domain'}}{$name}{$email} = $user;
$list_cache{'get_subscriber'}{$options->{'domain'}}{$name}{$email} = $user;
return $user;
}
## Returns an array of all users in User table hash for a given user
sub get_subscriber_by_bounce_address {
......
......@@ -280,7 +280,11 @@ sub mail_file {
'robot' => $robot,
'listname' => $listname,
'priority' => &Conf::get_robot_conf($robot,'sympa_priority'),
'sign_mode' => $sign_mode)) {
'sign_mode' => $sign_mode,
'use_bulk' => $data->{'use_bulk'},
)
)
{
return undef;
}
return 1;
......@@ -644,21 +648,22 @@ sub sending {
my $verpfeature = ($verp eq 'on');
if ($use_bulk){ # in that case use bulk tables to prepare message distribution
my $bulk_code = &Bulk::store('msg' => $messageasstring,
'rcpts' => $rcpt,
'from' => $from,
'robot' => $robot,
'listname' => $listname,
'priority_message' => $priority_message,
'priority_packet' => $priority_packet,
'delivery_date' => $delivery_date,
'verp' => $verpfeature);
unless (defined $bulk_code) {
&do_log('err', 'Failed to store message for list %s', $listname);
&List::send_notify_to_listmaster('bulk_error', $robot, {'listname' => $listname});
return undef;
}
my $bulk_code = &Bulk::store('msg' => $messageasstring,
'rcpts' => $rcpt,
'from' => $from,
'robot' => $robot,
'listname' => $listname,
'priority_message' => $priority_message,
'priority_packet' => $priority_packet,
'delivery_date' => $delivery_date,
'verp' => $verpfeature);
unless (defined $bulk_code) {
&do_log('err', 'Failed to store message for list %s', $listname);
&List::send_notify_to_listmaster('bulk_error', $robot, {'listname' => $listname});
return undef;
}
}elsif(defined $send_spool) { # in context wwsympa.fcgi do note send message to reciepients but copy it to standard spool
$sympa_email = &Conf::get_robot_conf($robot, 'sympa');
$sympa_file = "$send_spool/T.$sympa_email.".time.'.'.int(rand(10000));
......
</
......@@ -3193,8 +3193,67 @@ sub unlock {
return 1;
}
## input a string