You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by mm...@apache.org on 2015/10/15 20:40:21 UTC
svn commit: r1708863 - in /spamassassin/trunk:
lib/Mail/SpamAssassin/PerMsgStatus.pm lib/Mail/SpamAssassin/Util.pm
t/header_utf8.t t/prefs_include.t t/reportheader_8bit.t
Author: mmartinec
Date: Thu Oct 15 18:40:21 2015
New Revision: 1708863
URL: http://svn.apache.org/viewvc?rev=1708863&view=rev
Log:
Bug 7253: X-Spam-Report incorrectly mime-encodes multiline report in header, violating RFC 2047
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
spamassassin/trunk/t/header_utf8.t
spamassassin/trunk/t/prefs_include.t
spamassassin/trunk/t/reportheader_8bit.t
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?rev=1708863&r1=1708862&r2=1708863&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Thu Oct 15 18:40:21 2015
@@ -60,7 +60,8 @@ use Encode;
use Mail::SpamAssassin::Constants qw(:sa);
use Mail::SpamAssassin::AsyncLoop;
use Mail::SpamAssassin::Conf;
-use Mail::SpamAssassin::Util qw(untaint_var uri_list_canonicalize idn_to_ascii);
+use Mail::SpamAssassin::Util qw(untaint_var base64_encode idn_to_ascii
+ uri_list_canonicalize);
use Mail::SpamAssassin::Timeout;
use Mail::SpamAssassin::Logger;
@@ -1005,7 +1006,7 @@ sub _get_added_headers {
foreach my $hf_ref (@{$self->{conf}->{$which}}) {
my($hfname, $hfbody) = @$hf_ref;
my $line = $self->_process_header($hfname,$hfbody);
- $line = $self->qp_encode_header($line);
+ $line = $self->mime_encode_header($line);
$str .= "X-Spam-$hfname: $line\n";
}
return $str;
@@ -1269,27 +1270,53 @@ sub rewrite_no_report_safe {
return $newmsg.$self->{msg}->get_pristine_body();
}
-sub qp_encode_header {
+# encode a header field body into ASCII as per RFC 2047
+#
+sub mime_encode_header {
my ($self, $text) = @_;
- # return unchanged if there are no 8-bit characters
- return $text if $text !~ tr/\x00-\x7F//c;
+ utf8::encode($text) if utf8::is_utf8($text);
-# my $cs = $self->{conf}->{report_charset} || "UTF-8";
- my $cs = "UTF-8";
+ my $result = '';
+ for my $line (split(/^/, $text)) {
- my @hexchars = split('', '0123456789abcdef');
- my $ord;
- local $1;
- $text =~ s{([\x80-\xff])}{
- $ord = ord $1;
- '='.$hexchars[($ord & 0xf0) >> 4].$hexchars[$ord & 0x0f]
- }ges;
+ if ($line =~ /^[\x09\x20-\x7E]*\r?\n\z/s) {
+ $result .= $line; # no need for encoding
+
+ } else {
+ my $prefix = '';
+ my $suffix = '';
- $text = '=?'.$cs.'?Q?'.$text.'?=';
+ local $1;
+ if ($line =~ s/( (?: ^ | [ \t] ) [\x09\x20-\x7E]* (?: \r?\n )? ) \z//xs) {
+ $suffix = $1;
+ } elsif ($line =~ s/(\r?\n)\z//s) {
+ $suffix = $1;
+ }
+
+ if ($line =~ s/^ ( [\x09\x20-\x7E]* (?: [ \t] | \z ) )//xs) {
+ $prefix = $1;
+ }
+
+ if ($line eq '') {
+ $result .= $prefix . $suffix;
+ } else {
+ my $qp_enc_count = $line =~ tr/=?_\x00-\x1F\x7F-\xFF//;
+ if (length($line) + $qp_enc_count*2 <= 4 * int(length($line)+2)/3) {
+ # RFC 2047: Upper case should be used for hex digits A through F
+ $line =~ s{ ( [=?_\x00-\x20\x7F-\xFF] ) }
+ { $1 eq ' ' ? '_' : sprintf("=%02X", ord $1) }xges;
+ $result .= $prefix . '=?UTF-8?Q?' . $line;
+ } else {
+ $result .= $prefix . '=?UTF-8?B?' . base64_encode($line);
+ }
+ $result .= '?=' . $suffix;
+ }
+ }
+ }
- dbg("markup: encoding header in $cs: $text");
- return $text;
+ dbg("markup: mime_encode_header: %s", $result);
+ return $result;
}
sub _process_header {
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm?rev=1708863&r1=1708862&r2=1708863&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm Thu Oct 15 18:40:21 2015
@@ -59,7 +59,8 @@ BEGIN {
@ISA = qw(Exporter);
@EXPORT = ();
- @EXPORT_OK = qw(&local_tz &base64_decode &untaint_var &untaint_file_path
+ @EXPORT_OK = qw(&local_tz &base64_decode &base64_encode
+ &untaint_var &untaint_file_path
&exit_status_str &proc_status_ok &am_running_on_windows
&reverse_ip_address &decode_dns_question_entry
&secure_tmpfile &secure_tmpdir &uri_list_canonicalize
@@ -873,7 +874,7 @@ sub base64_encode {
local $_ = shift;
if (HAS_MIME_BASE64) {
- return MIME::Base64::encode_base64($_);
+ return MIME::Base64::encode_base64($_,'');
}
$_ = pack("u57", $_);
Modified: spamassassin/trunk/t/header_utf8.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/header_utf8.t?rev=1708863&r1=1708862&r2=1708863&view=diff
==============================================================================
--- spamassassin/trunk/t/header_utf8.t (original)
+++ spamassassin/trunk/t/header_utf8.t Thu Oct 15 18:40:21 2015
@@ -5,13 +5,13 @@ use SATest; sa_t_init("header_utf8.t");
use constant TEST_ENABLED => ($] > 5.010001);
-use Test; BEGIN { plan tests => (TEST_ENABLED ? 44 : 0) };
+use Test; BEGIN { plan tests => (TEST_ENABLED ? 154 : 0) };
exit unless (TEST_ENABLED);
# ---------------------------------------------------------------------------
-%patterns = (
+%mypatterns = (
q{/ LT_RPATH /} => 'LT_RPATH',
q{/ LT_ENVFROM /} => 'LT_ENVFROM',
q{/ LT_FROM /} => 'LT_FROM',
@@ -31,8 +31,19 @@ exit unless (TEST_ENABLED);
q{/ LT_NOTE /} => 'LT_NOTE',
q{/ LT_UTF8SMTP_ANY /} => 'LT_UTF8SMTP_ANY',
q{/ LT_SPLIT_UTF8_SUBJ /} => 'LT_SPLIT_UTF8_SUBJ',
- q{/ USER_IN_WHITELIST /} => 'USER_IN_WHITELIST',
- q{/(?m) LT_ANY_CHARS \s*En-tête contient caractères$/} => 'LT_ANY_CHARS',
+ q{/ USER_IN_BLACKLIST /} => 'USER_IN_BLACKLIST',
+);
+
+%mypatterns_utf8 = ( # as it appears in a report body
+ q{/(?m)^ 0\.0 LT_ANY_CHARS \s*En-tête contient caractères$/} => 'LT_ANY_CHARS utf8',
+);
+
+%mypatterns_mime_qp = ( # as it appears in a mail header section
+ q{/(?m)^\t\* 0\.0 LT_ANY_CHARS =\?UTF-8\?Q\?En-t=C3=AAte_contient_caract=C3=A8res\?=$/} => 'LT_ANY_CHARS mime encoded',
+);
+
+%mypatterns_mime_b64 = ( # as it appears in a mail header section
+ q{/(?m)^\t\* 0\.0 LT_ANY_CHARS =\?UTF-8\?B\?5a2X56ym6KKr5YyF5ZCr5Zyo5raI5oGv5oql5aS06YOo5YiG\?=$/} => 'LT_ANY_CHARS mime encoded',
);
%anti_patterns = (
@@ -40,9 +51,9 @@ exit unless (TEST_ENABLED);
# q{/ INVALID_MSGID /} => 'INVALID_MSGID',
);
-my $localrules = <<'END';
+my $myrules = <<'END';
add_header all AuthorDomain _AUTHORDOMAIN_
- whitelist_from Marilù.Gioffré@esempio-università .it
+ blacklist_from Marilù.Gioffré@esempio-università .it
header LT_UTF8SMTP_ANY Received =~ /\bwith\s*UTF8SMTPS?A?\b/mi
score LT_UTF8SMTP_ANY -0.1
header LT_RPATH Return-Path:addr =~ /^Marilù\.Gioffré\@esempio-università \.it\z/
@@ -82,18 +93,77 @@ my $localrules = <<'END';
header LT_NOTE X-Note =~ /^The above.*char =C5 =BE is invalid, .*wild$/m
score LT_NOTE 0.01
header LT_ANY_CHARS From =~ /./
- score LT_ANY_CHARS 0.01
- describe LT_ANY_CHARS En-tête contient caractères
+ score LT_ANY_CHARS 0.001
+ describe LT_ANY_CHARS Header contains characters
+ lang fr describe LT_ANY_CHARS En-tête contient caractères
+ # sorry, Google translate:
+ lang zh describe LT_ANY_CHARS å符被å
å«å¨æ¶æ¯æ¥å¤´é¨å
END
-tstlocalrules ($localrules . '
+$ENV{LANG} = 'fr_CH.UTF-8';
+$ENV{LC_ALL} = 'fr_CH.UTF-8';
+
+#--- normalize_charset 1
+
+tstlocalrules ($myrules . '
+ report_safe 0
+ normalize_charset 1
+');
+%patterns = (%mypatterns, %mypatterns_mime_qp);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
+ok_all_patterns();
+
+tstlocalrules ($myrules . '
+ report_safe 1
+ normalize_charset 1
+');
+%patterns = (%mypatterns, %mypatterns_utf8);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
+ok_all_patterns();
+
+tstlocalrules ($myrules . '
+ report_safe 2
+ normalize_charset 1
+');
+%patterns = (%mypatterns, %mypatterns_utf8);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
+ok_all_patterns();
+
+#--- normalize_charset 0
+
+tstlocalrules ($myrules . '
+ report_safe 0
normalize_charset 0
');
-sarun ("-L -t < data/nice/unicode1", \&patterns_run_cb);
+%patterns = (%mypatterns, %mypatterns_mime_qp);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
ok_all_patterns();
-tstlocalrules ($localrules . '
+tstlocalrules ($myrules . '
+ report_safe 1
+ normalize_charset 0
+');
+%patterns = (%mypatterns, %mypatterns_utf8);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
+ok_all_patterns();
+
+tstlocalrules ($myrules . '
+ report_safe 2
+ normalize_charset 0
+');
+%patterns = (%mypatterns, %mypatterns_utf8);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
+ok_all_patterns();
+
+#--- base64 encoded-words
+
+$ENV{LANG} = 'zh_CN.UTF-8';
+$ENV{LC_ALL} = 'zh_CN.UTF-8';
+
+tstlocalrules ($myrules . '
+ report_safe 0
normalize_charset 1
');
-sarun ("-L -t < data/nice/unicode1", \&patterns_run_cb);
+%patterns = (%mypatterns, %mypatterns_mime_b64);
+sarun ("-L < data/nice/unicode1", \&patterns_run_cb);
ok_all_patterns();
Modified: spamassassin/trunk/t/prefs_include.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/prefs_include.t?rev=1708863&r1=1708862&r2=1708863&view=diff
==============================================================================
--- spamassassin/trunk/t/prefs_include.t (original)
+++ spamassassin/trunk/t/prefs_include.t Thu Oct 15 18:40:21 2015
@@ -2,7 +2,7 @@
use lib '.'; use lib 't';
use SATest; sa_t_init("prefs_include");
-use Test; BEGIN { plan tests => 2 };
+use Test; BEGIN { plan tests => 3 };
$ENV{'LANGUAGE'} = $ENV{'LC_ALL'} = 'C'; # a cheat, but we need the patterns to work
@@ -10,9 +10,9 @@ $ENV{'LANGUAGE'} = $ENV{'LC_ALL'} = 'C';
%patterns = (
- q{X-Spam-Report: =?UTF-8?Q? }, 'qp-encoded-hdr',
- q{ Invalid Date: header =ae =af =b0 foo }, 'qp-encoded-desc',
-
+ q{/(?m)^X-Spam-Report:\s*$/}, 'qp-encoded-hdr',
+ q{/(?m)^\t\*\s+[0-9.-]+ INVALID_DATE\s+Invalid Date: header =\?UTF-8\?B\?wq4gwq8gwrA=\?=$/}, 'qp-encoded-desc',
+ q{/(?m)^ [0-9.-]+ INVALID_DATE\s+Invalid Date: header ® ¯ °$/}, 'report-desc',
);
tstprefs ("
@@ -23,7 +23,7 @@ tstprefs ("
open (OUT, ">log/prefs_include.inc") or die "open log/prefs_include.inc failed";
print OUT "
report_safe 0
- describe INVALID_DATE Invalid Date: header \xae \xaf \xb0 foo
+ describe INVALID_DATE Invalid Date: header ® ¯ °
";
close OUT;
Modified: spamassassin/trunk/t/reportheader_8bit.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/reportheader_8bit.t?rev=1708863&r1=1708862&r2=1708863&view=diff
==============================================================================
--- spamassassin/trunk/t/reportheader_8bit.t (original)
+++ spamassassin/trunk/t/reportheader_8bit.t Thu Oct 15 18:40:21 2015
@@ -2,7 +2,7 @@
use lib '.'; use lib 't';
use SATest; sa_t_init("reportheader");
-use Test; BEGIN { plan tests => 2 };
+use Test; BEGIN { plan tests => 3 };
$ENV{'LANGUAGE'} = $ENV{'LC_ALL'} = 'C'; # a cheat, but we need the patterns to work
@@ -10,15 +10,16 @@ $ENV{'LANGUAGE'} = $ENV{'LC_ALL'} = 'C';
%patterns = (
- q{X-Spam-Report: =?UTF-8?Q? }, 'qp-encoded-hdr',
- q{ Invalid Date: header =ae =af =b0 foo }, 'qp-encoded-desc',
+ q{/(?m)^X-Spam-Report:\s*$/}, 'qp-encoded-hdr',
+ q{/(?m)^\t\*\s+[0-9.-]+ INVALID_DATE\s+Invalid Date: header =\?UTF-8\?B\?wq4gwq8gwrA=\?= foo$/}, 'qp-encoded-desc',
+ q{/(?m)^ [0-9.-]+ INVALID_DATE\s+Invalid Date: header ® ¯ ° foo$/}, 'report-desc',
);
tstprefs ("
$default_cf_lines
report_safe 0
- describe INVALID_DATE Invalid Date: header \xae \xaf \xb0 foo
+ describe INVALID_DATE Invalid Date: header ® ¯ ° foo
");
sarun ("-L -t < data/spam/001", \&patterns_run_cb);