You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by fe...@apache.org on 2004/03/09 07:04:10 UTC
svn commit: rev 9307 - in incubator/spamassassin/trunk: lib/Mail lib/Mail/SpamAssassin t
Author: felicity
Date: Mon Mar 8 22:04:09 2004
New Revision: 9307
Modified:
incubator/spamassassin/trunk/lib/Mail/SpamAssassin.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
incubator/spamassassin/trunk/t/stripmarkup.t
Log:
bug 3077: more work to speed up remove_spamassassin_markup... 'report_safe 0' now saves a pristine copy of the header to X-Spam-Prev-Header, and remove_sa_markup properly moves it back. it also keep backward compatibility for pre-3.0's rewrite headers, which unfortunately means the config is loaded. but now we only load the config when necessary (ie: if the message(s) are report_safe encapsulated, no config is read. ;) made a few other efficiency improvements, and fixed the stripmarkup test to deal with the new way the markup is removed.
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin.pm Mon Mar 8 22:04:09 2004
@@ -846,28 +846,28 @@
my $mbox = $mail_obj->get_mbox_seperator() || '';
dbg("Removing Markup");
- $self->init(1);
+
+ # Go looking for a "report_safe" encapsulated message. Abort out ASAP
+ # if we have definitive proof it's not an encapsulated message.
my $ct = $mail_obj->get_header("Content-Type") || '';
- if ( $ct
- && $ct =~ m!^\s*multipart/mixed;\s+boundary\s*=\s*["']?(.+?)["']?(?:;|$)!i )
- {
+ if ( $ct =~ m!^\s*multipart/mixed;\s+boundary\s*=\s*["']?(.+?)["']?(?:;|$)!i ) {
# Ok, this is a possible encapsulated message, search for the
# appropriate mime part and deal with it if necessary.
my $boundary = "\Q$1\E";
- my @msg = split(/^/,$mail_obj->get_pristine());
+ my @msg = split(/^/,$mail_obj->get_pristine_body());
my $flag = 0;
$ct = '';
my $cd = '';
for ( my $i = 0 ; $i <= $#msg ; $i++ ) {
- # only look at mime headers
+ # only look at mime part headers
next unless ( $msg[$i] =~ /^--$boundary$/ || $flag );
if ( $msg[$i] =~ /^\s*$/ ) { # end of mime header
# Ok, we found the encapsulated piece ...
- if ($ct =~ m@(?:message/rfc822|text/plain);\s+x-spam-type=original@ ||
+ if ($ct =~ m@^(?:message/rfc822|text/plain);\s+x-spam-type=original@ ||
($ct eq "message/rfc822" &&
$cd eq $self->{'encapsulated_content_description'}))
{
@@ -883,7 +883,7 @@
}
}
- # Ok, we're done. Return the message.
+ # Ok, we're done. Return the rewritten message.
return join('', $mbox, @msg);
}
@@ -904,38 +904,23 @@
}
}
+ # Ok, if we got here, the message wasn't a report_safe encapsulated message.
+ # So treat it like a "report_safe 0" message.
my $hdrs = $mail_obj->get_pristine_header();
+ my $body = $mail_obj->get_pristine_body();
# remove DOS line endings
$hdrs =~ s/\r//gs;
- # de-break lines on SpamAssassin-modified headers.
- 1 while $hdrs =~ s/(\n(?:X-Spam|Subject)[^\n]+?)\n[ \t]+/$1 /gs;
-
- # reinstate the old content type
- if ($hdrs =~ /^X-Spam-Prev-Content-Type: /m) {
- $hdrs =~ s/\nContent-Type: [^\n]*?\n/\n/gs;
- $hdrs =~ s/\nX-Spam-Prev-(Content-Type: [^\n]*\n)/\n$1/gs;
-
- # remove embedded spaces where they shouldn't be; a common problem
- $hdrs =~ s/(Content-Type: .*?boundary=\".*?) (.*?\".*?\n)/$1$2/gs;
- }
-
- # reinstate the old content transfer encoding
- if ($hdrs =~ /^X-Spam-Prev-Content-Transfer-Encoding: /m) {
- $hdrs =~ s/\nContent-Transfer-Encoding: [^\n]*?\n/\n/gs;
- $hdrs =~ s/\nX-Spam-Prev-(Content-Transfer-Encoding: [^\n]*\n)/\n$1/gs;
- }
-
- # reinstate the return-receipt-to header
- if ($hdrs =~ /^X-Spam-Prev-Return-Receipt-To: /m) {
- $hdrs =~ s/\nX-Spam-Prev-(Return-Receipt-To: [^\n]*\n)/\n$1/gs;
- }
+ # unfold SA added headers, but not X-Spam-Prev headers ...
+ 1 while $hdrs =~ s/(\nX-Spam-(?!Prev).+?)\n[ \t]+/$1 /g;
- # remove the headers we added
- 1 while $hdrs =~ s/\nX-Spam-[^\n]*?\n/\n/gs;
+###########################################################################
+ # Backward Compatibilty, pre 3.0.x.
- foreach my $header ( keys %{$self->{conf}->{rewrite_header}} ) {
+ # deal with rewritten headers w/out X-Spam-Prev- versions ...
+ $self->init(1);
+ foreach my $header ( keys %{$self->{conf}->{rewrite_header}} ) {
dbg ("Removing markup in $header");
if ($header eq 'Subject') {
my $tag = $self->{conf}->{rewrite_header}->{'Subject'};
@@ -949,40 +934,27 @@
}
}
- # ok, next, the report.
- # This is a little tricky since we can have either 0, 1 or 2 reports;
- # 0 for the non-spam case, 1 for normal filtering, and 2 for -t (where
- # an extra report is appended at the end of the mail).
-
- my @newbody = ();
- my $inreport = 0;
- foreach $_ (@{$mail_obj->get_body()})
- {
- s/\r?$//; # DOS line endings
-
- if (/^SPAM: ----/ && $inreport == 0) {
- # we've just entered a report. If there's a blank line before the
- # report, get rid of it...
- if ($#newbody > 0 && $newbody[$#newbody-1] =~ /^$/) {
- pop (@newbody);
- }
- # and skip on to the next line...
- $inreport = 1; next;
- }
-
- if ($inreport && /^$/) {
- # blank line at end of report; skip it. Also note that we're
- # now out of the report.
- $inreport = 0; next;
- }
-
- # finally, if we're not in the report, add it to the body array
- if (!$inreport) {
- push (@newbody, $_);
+ # Now deal with report cleansing from 2.4x and previous.
+ # possibly a blank line, "SPAM: ----.+", followed by "SPAM: stuff" lines,
+ # followed by another "SPAM: ----.+" line, followed by a blank line.
+ 1 while ($body =~ s/^\n?SPAM: ----.+\n(?:SPAM:.*\n)*SPAM: ----.+\n\n//);
+###########################################################################
+
+ # 3.0 version -- revert from X-Spam-Prev to original ...
+ while ($hdrs =~ s/^X-Spam-Prev-(([^:]+:)\s*(?:.*\n(?:\s+\S.*\n)*))//mg) {
+ my($hdr, $name) = ($1,$2);
+
+ # If the rewritten version doesn't exist, we should deal with it anyway...
+ unless ($hdrs =~ s/^$name\s*(?:.*\n(?:\s+\S.*\n)*)$/$hdr\n/m) {
+ $hdrs =~ s/\n\n/\n$hdr\n\n/;
}
}
- return join ('', $mbox, $hdrs, @newbody);
+ # remove any other X-Spam headers we added, will be unfolded
+ 1 while $hdrs =~ s/\nX-Spam-.*\n/\n/g;
+
+ # Put the whole thing back together ...
+ return join ('', $mbox, $hdrs, $body);
}
###########################################################################
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Mon Mar 8 22:04:09 2004
@@ -817,13 +817,18 @@
=cut
if ( $key eq 'rewrite_header' ) {
- # could check $hdr to be /subject|from|to/, but rest of the
- # code already handles that ...
my($hdr, $string) = split(/\s+/, $value, 2);
$hdr = ucfirst(lc($hdr));
- $string =~ tr/()/[]/;
- $self->{rewrite_header}->{$hdr} = $string;
- next;
+
+ # We only deal with From, Subject, and To ...
+ if ($hdr =~ /^(?:From|Subject|To)$/) {
+ $string =~ tr/()/[]/;
+ $self->{rewrite_header}->{$hdr} = $string;
+ next;
+ }
+
+ # if we get here, note the issue, then we'll fail through for an error.
+ dbg("rewrite_header: ignoring $hdr, not From, Subject, or To");
}
=item fold_headers { 0 | 1 } (default: 1)
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Mon Mar 8 22:04:09 2004
@@ -232,10 +232,11 @@
=item get_pristine_header()
Returns pristine headers of the message. If no specific header name
-is given as a parameter (case-insensitive), then all headers will
-be returned. If called in an array context, an array will be returned
-with each header (specific or all) in a different element. In a scalar
-context, either all of the headers are returned as a scalar, or the last
+is given as a parameter (case-insensitive), then all headers will be
+returned as a scalar, including the blank line at the end of the headers.
+
+If called in an array context, an array will be returned with each
+specific header in a different element. In a scalar context, the last
specific header is returned.
ie: If 'Subject' is specified as the header, and there are 2 Subject
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Mon Mar 8 22:04:09 2004
@@ -768,11 +768,15 @@
if($self->{is_spam}) {
# Deal with header rewriting
- while ( my($header, $value) = each %{$self->{conf}->{rewrite_header}}) {
- unless ( $header =~ /^(?:Subject|From|To)$/ ) {
- dbg("rewrite: ignoring $header = $value");
- next;
- }
+ foreach ( @pristine_headers ) {
+ # if we're not going to do a rewrite, skip this header!
+ next if (!/^(?:From|Subject|To):/i || !exists $self->{conf}->{rewrite_header}->{$1});
+
+ # pop the original version onto the end of the header array
+ push(@pristine_headers, "X-Spam-Prev-$_");
+
+ # prep the rewrite of the original
+ my($header, $value) = %{$self->{conf}->{rewrite_header}};
# Figure out the rewrite piece
my $tag = $self->_replace_tags($value);
@@ -781,12 +785,7 @@
# The tag should be a comment for this header ...
$tag = "($tag)" if ( $header =~ /^(?:From|To)$/ );
- # Go ahead and markup the headers
- foreach ( @pristine_headers ) {
- # skip non-correct-header or headers that are already tagged
- next if ( !/^${header}:/i );
- s/^([^:]+:[ ]*)(?:\Q${tag}\E )?/$1${tag} /i;
- }
+ s/^([^:]+:[ ]*)(?:\Q${tag}\E )?/$1${tag} /i;
}
$addition = 'headers_spam';
Modified: incubator/spamassassin/trunk/t/stripmarkup.t
==============================================================================
--- incubator/spamassassin/trunk/t/stripmarkup.t (original)
+++ incubator/spamassassin/trunk/t/stripmarkup.t Mon Mar 8 22:04:09 2004
@@ -11,6 +11,7 @@
q{ Content-Type: text/html }, 'contenttype',
q{
Sender: pertand@email.mondolink.com
+ Content-Type: text/html
<HTML></P>
}, 'startofbody',