You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by qu...@apache.org on 2004/08/03 04:47:32 UTC
svn commit: rev 35614 - in spamassassin/trunk: lib/Mail/SpamAssassin lib/Mail/SpamAssassin/Message rules
Author: quinlan
Date: Mon Aug 2 19:47:32 2004
New Revision: 35614
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/rules/20_head_tests.cf
spamassassin/trunk/rules/20_meta_tests.cf
spamassassin/trunk/rules/50_scores.cf
Log:
various performance improvements, long header test
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm Mon Aug 2 19:47:32 2004
@@ -69,13 +69,12 @@
# *less* these well-known headers; that way we can pick up spammers' tracking
# headers (which are obviously not well-known in advance!).
+# Received is handled specially
$IGNORED_HDRS = qr{(?: (?:X-)?Sender # misc noise
|Delivered-To |Delivery-Date
|(?:X-)?Envelope-To
|X-MIME-Auto[Cc]onverted |X-Converted-To-Plain-Text
- |Received # handled specially
-
|Subject # not worth a tiny gain vs. to db size increase
# Date: can provide invalid cues if your spam corpus is
@@ -490,39 +489,43 @@
sub tokenize_headers {
my ($self, $msg) = @_;
- my $hdrs = $msg->get_all_headers() . $msg->get_all_metadata();
-
my %parsed = ();
- # we don't care about whitespace; so fix continuation lines to make the next
- # bit easier
- $hdrs =~ s/\n[ \t]+/ /gs;
-
- # first, keep a copy of Received hdrs, so we can strip down to last 2
- my @rcvdlines = ($hdrs =~ /^Received: [^\n]*$/gim);
-
- # and now delete lines for headers we don't want (incl all Receiveds)
- $hdrs =~ s/^${IGNORED_HDRS}: [^\n]*$//gim;
+ my %user_ignore;
+ $user_ignore{$_} = 1 for @{$self->{main}->{conf}->{bayes_ignore_headers}};
- if (IGNORE_MSGID_TOKENS) { $hdrs =~ s/^Message-I[dD]: [^\n]*$//gim;}
+ # get headers in array context
+ my @hdrs;
+ my @rcvdlines;
+ for ($msg->get_all_headers()) {
+ # first, keep a copy of Received headers, so we can strip down to last 2
+ if (/^Received:/i) {
+ push(@rcvdlines, $_);
+ next;
+ }
+ # and now skip lines for headers we don't want (including all Received)
+ next if /^${IGNORED_HDRS}:/i;
+ next if IGNORE_MSGID_TOKENS && /^Message-ID:/i;
+ push(@hdrs, $_);
+ }
+ push(@hdrs, $msg->get_all_metadata());
# and re-add the last 2 received lines: usually a good source of
# spamware tokens and HELO names.
- if ($#rcvdlines >= 0) { $hdrs .= "\n".$rcvdlines[$#rcvdlines]; }
- if ($#rcvdlines >= 1) { $hdrs .= "\n".$rcvdlines[$#rcvdlines-1]; }
-
- # remove user-specified headers here, after Received, in case they
- # want to ignore that too
- foreach my $conf (@{$self->{main}->{conf}->{bayes_ignore_headers}}) {
- $hdrs =~ s/^\Q${conf}\E: [^\n]*$//gim;
- }
+ if ($#rcvdlines >= 0) { push(@hdrs, $rcvdlines[$#rcvdlines]); }
+ if ($#rcvdlines >= 1) { push(@hdrs, $rcvdlines[$#rcvdlines-1]); }
- while ($hdrs =~ /^(\S+): ([^\n]*)$/gim) {
- my $hdr = $1;
- my $val = $2;
+ for (@hdrs) {
+ next unless /\S/;
+ my ($hdr, $val) = split(/:/, $_, 2);
+ $val ||= '';
+
+ # remove user-specified headers here, after Received, in case they
+ # want to ignore that too
+ next if exists $user_ignore{$hdr};
# special tokenization for some headers:
- if ($hdr =~ /^(?:|X-|Resent-)Message-I[dD]$/) {
+ if ($hdr =~ /^(?:|X-|Resent-)Message-Id$/i) {
$val = $self->pre_chew_message_id ($val);
}
elsif (PRE_CHEW_ADDR_HEADERS && $hdr =~ /^(?:|X-|Resent-)
@@ -544,8 +547,7 @@
}
if (MAP_HEADERS_MID) {
- if ($hdr eq 'In-Reply-To' || $hdr eq 'References' || $hdr =~ /^Message-Id/i)
- {
+ if ($hdr =~ /^(?:In-Reply-To|References|Message-ID)$/i) {
$parsed{"*MI"} = $val;
}
}
@@ -555,7 +557,7 @@
}
}
if (MAP_HEADERS_USERAGENT) {
- if ($hdr =~ /^(?:X-Mailer|User-Agent)$/) {
+ if ($hdr =~ /^(?:X-Mailer|User-Agent)$/i) {
$parsed{"*UA"} = $val;
}
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm Mon Aug 2 19:47:32 2004
@@ -35,6 +35,8 @@
# These are generic constants that may be used across several modules
@SA_VARS = qw(
META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY MBX_SEPARATOR
+ MAX_BODY_LINE_LENGTH MAX_HEADER_KEY_LENGTH MAX_HEADER_VALUE_LENGTH
+ MAX_HEADER_LENGTH
);
%EXPORT_TAGS = (
@@ -175,5 +177,17 @@
# $2 = size of message in bytes (int)
# $3 = message status - binary (hex)
# $4 = message ID (hex)
+
+# ---------------------------------------------------------------------------
+# values used for internal message representations
+
+# maximum byte length of lines in the body
+use constant MAX_BODY_LINE_LENGTH => 2048;
+# maximum byte length of a header key
+use constant MAX_HEADER_KEY_LENGTH => 256;
+# maximum byte length of a header value including continued lines
+use constant MAX_HEADER_VALUE_LENGTH => 8192;
+# maximum byte length of entire header
+use constant MAX_HEADER_LENGTH => 65536;
1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm Mon Aug 2 19:47:32 2004
@@ -184,23 +184,22 @@
sub _check_recipients {
my ($self) = @_;
- my @address;
+ my @inputs;
# ToCc: pseudo-header works best, but sometimes Bcc: is better
for ('ToCc', 'Bcc') {
my $to = $self->get($_); # get recipients
$to =~ s/\(.*?\)//g; # strip out the (comments)
- @address = ($to =~ m/([\w.=-]+\@\w+(?:[\w.-]+\.)+\w+)/g);
- last if scalar(@address) >= TOCC_SIMILAR_COUNT;
+ @inputs = ($to =~ m/([\w.=-]+\@\w+(?:[\w.-]+\.)+\w+)/g);
+ last if scalar(@inputs) >= TOCC_SIMILAR_COUNT;
}
# remove duplicate addresses only when they appear next to each other
- my @tmp = @address;
+ my @address;
my $previous = '';
- @address = ();
- foreach my $current (@tmp) {
- next if (lc($current) eq lc($previous));
- push(@address, ($previous = $current));
+ while (my $current = shift @inputs) {
+ push(@address, ($previous = $current)) if lc($current) ne lc($previous);
+ last if @address == 256;
}
# ideas that had both poor S/O ratios and poor hit rates:
@@ -2662,22 +2661,18 @@
}
sub check_pyzor {
- my ($self, $fulltext) = @_;
+ my ($self, $full) = @_;
return 0 unless ($self->is_pyzor_available());
return 0 if ($self->{already_checked_pyzor});
$self->{already_checked_pyzor} = 1;
- # note: we don't use $fulltext. instead we get the raw message,
- # unfiltered, for pyzor to check. ($fulltext removes MIME
- # parts etc.)
- my $full = $self->{msg}->get_pristine();
- return $self->pyzor_lookup (\$full);
+ return $self->pyzor_lookup($full);
}
sub check_dcc {
- my ($self, $fulltext) = @_;
+ my ($self, $full) = @_;
my $have_dccifd = $self->is_dccifd_available();
return 0 unless ($have_dccifd || $self->is_dcc_available() );
@@ -2691,14 +2686,11 @@
$_ = $self->get('X-DCC-(?:[^:]+-)?Metrics');
return 1 if /bulk/;
- # note: we don't use $fulltext. instead we get the raw message,
- # unfiltered, for DCC to check. ($fulltext removes MIME
- # parts etc.)
- my $full = $self->{msg}->get_pristine();
if ($have_dccifd) {
- return $self->dccifd_lookup (\$full);
- } else {
- return $self->dcc_lookup (\$full);
+ return $self->dccifd_lookup($full);
+ }
+ else {
+ return $self->dcc_lookup($full);
}
}
@@ -3387,5 +3379,11 @@
}
###########################################################################
+
+sub check_for_long_header {
+ my ($self) = @_;
+
+ return defined $self->{msg}->{'truncated_header'};
+}
1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Mon Aug 2 19:47:32 2004
@@ -54,8 +54,6 @@
@ISA = qw(Mail::SpamAssassin::Message::Node);
-use constant MAX_BODY_LINE_LENGTH => 2048;
-
# ---------------------------------------------------------------------------
=item new()
@@ -170,11 +168,21 @@
# Ok, there's a header here, let's go ahead and add it in.
if ($header) {
- my ( $key, $value ) = split ( /:\s*/, $header, 2 );
+ # Yes, the /s is needed to match \n too.
+ my ($key, $value) = split (/:\s*(?=.)/s, $header, 2);
# If it's not a valid header (aka: not in the form "foo: bar"), skip it.
if (defined $value) {
- $self->header( $key, $value );
+ # limit the length of the pairs we store
+ if (length($key) > MAX_HEADER_KEY_LENGTH) {
+ $key = substr($key, 0, MAX_HEADER_KEY_LENGTH);
+ $self->{'truncated_header'} = 1;
+ }
+ if (length($value) > MAX_HEADER_VALUE_LENGTH) {
+ $value = substr($value, 0, MAX_HEADER_VALUE_LENGTH);
+ $self->{'truncated_header'} = 1;
+ }
+ $self->header($key, $value);
}
}
@@ -481,11 +489,11 @@
}
if ($in_body) {
- # we run into a perl bug if the lines are astronomically long (probably due
- # to lots of regexp backtracking); so cut short any individual line over
- # MAX_BODY_LINE_LENGTH bytes in length. This can wreck HTML totally -- but
- # IMHO the only reason a luser would use MAX_BODY_LINE_LENGTH-byte lines is
- # to crash filters, anyway.
+ # we run into a perl bug if the lines are astronomically long (probably
+ # due to lots of regexp backtracking); so cut short any individual line
+ # over MAX_BODY_LINE_LENGTH bytes in length. This can wreck HTML
+ # totally -- but IMHO the only reason a luser would use
+ # MAX_BODY_LINE_LENGTH-byte lines is to crash filters, anyway.
while (length ($_) > MAX_BODY_LINE_LENGTH) {
push (@{$part_array}, substr($_, 0, MAX_BODY_LINE_LENGTH)."\n");
substr($_, 0, MAX_BODY_LINE_LENGTH) = '';
@@ -605,9 +613,9 @@
}
# whitespace handling (warning: small changes have large effects!)
- $text =~ s/\n+\s*\n+/\f/gs; # double newlines => form feed
- $text =~ tr/ \t\n\r\x0b\xa0/ /s; # whitespace => space
- $text =~ tr/\f/\n/; # form feeds => newline
+ $text =~ s/\n+\s*\n+/\f/gs; # double newlines => form feed
+ $text =~ tr/ \t\n\r\x0b\xa0/ /s; # whitespace => space
+ $text =~ tr/\f/\n/; # form feeds => newline
# warn "JMD $text";
@@ -803,9 +811,9 @@
}
my @ret = ();
foreach my $key (sort keys %{$self->{metadata}->{strings}}) {
- push (@ret, $key, ": ", $self->{metadata}->{strings}->{$key}, "\n");
+ push (@ret, "$key: " . $self->{metadata}->{strings}->{$key} . "\n");
}
- return join ("", @ret);
+ return (wantarray ? @ret : join('', @ret));
}
# ---------------------------------------------------------------------------
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm Mon Aug 2 19:47:32 2004
@@ -38,6 +38,7 @@
use bytes;
use Mail::SpamAssassin;
+use Mail::SpamAssassin::Constants qw(:sa);
use Mail::SpamAssassin::HTML;
use MIME::Base64;
use MIME::QuotedPrint;
@@ -543,6 +544,8 @@
=cut
+# TODO: this could be made much faster by only processing all headers
+# when called in array context, otherwise just do one header
sub get_header {
my ($self, $hdr, $raw) = @_;
$raw ||= 0;
@@ -585,27 +588,46 @@
=cut
+# build it and it will not bomb
sub get_all_headers {
my ($self, $raw, $include_mbox) = @_;
$raw ||= 0;
$include_mbox ||= 0;
- my %cache = ();
my @lines = ();
- # we're guaranteed that get_header() will return a non-undef value here,
- # so don't bother trying to deal with that possibility.
- foreach ( @{$self->{header_order}} ) {
- push(@lines, "$_: ".($self->get_header($_,$raw))[$cache{$_}++]);
+ # precalculate destination positions based on order of appearance
+ my $i = 0;
+ my %locations;
+ for my $k (@{$self->{header_order}}) {
+ push(@{$locations{lc($k)}}, $i++);
+ }
+
+ # process headers in order of first appearance
+ my $header;
+ my $size = 0;
+ HEADER: for my $name (sort { $locations{$a}->[0] <=> $locations{$b}->[0] }
+ keys %locations)
+ {
+ # get all same-name headers and poke into correct position
+ my $positions = $locations{$name};
+ for my $contents ($self->get_header($name, $raw)) {
+ my $position = shift @{$positions};
+ $size += length($name) + length($contents) + 2;
+ if ($size > MAX_HEADER_LENGTH) {
+ $self->{'truncated_header'} = 1;
+ last HEADER;
+ }
+ $lines[$position] = $self->{header_order}->[$position] . ": $contents";
+ }
}
+ # skip undefined lines if we truncated
+ @lines = grep { defined $_ } @lines if $self->{'truncated_header'};
+
splice @lines, 0, 0, $self->{mbox_sep} if ( $include_mbox && exists $self->{mbox_sep} );
- if (wantarray) {
- return @lines;
- } else {
- return join ('', @lines);
- }
+ return wantarray ? @lines : join ('', @lines);
}
# ---------------------------------------------------------------------------
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Mon Aug 2 19:47:32 2004
@@ -153,8 +153,7 @@
my $bodytext = $self->get_decoded_body_text_array();
- my $fulltext = join ('', $self->{msg}->get_all_headers(1), "\n",
- $self->{msg}->get_pristine_body());
+ my $fulltext = $self->{msg}->get_pristine();
# use $bodytext here because $decoded is too stripped
# TVD: leave it up to get_uri_list to do the right thing ...
Modified: spamassassin/trunk/rules/20_head_tests.cf
==============================================================================
--- spamassassin/trunk/rules/20_head_tests.cf (original)
+++ spamassassin/trunk/rules/20_head_tests.cf Mon Aug 2 19:47:32 2004
@@ -24,6 +24,9 @@
require_version @@VERSION@@
+header HEAD_LONG eval:check_for_long_header()
+describe HEAD_LONG Message headers are very long
+
header NO_REAL_NAME From =~ /^["\s]*\<?\S+\@\S+\>?\s*$/
describe NO_REAL_NAME From: does not include a real name
@@ -33,11 +36,13 @@
header FROM_STARTS_WITH_NUMS From =~ /^\d\d/
describe FROM_STARTS_WITH_NUMS From: starts with nums
-header FROM_HAS_MIXED_NUMS From =~ /\d+[a-z]+\d+\S*\@/i
+# note: anchored for speed
+header FROM_HAS_MIXED_NUMS From =~ /(?:^|\D)\d+[a-z]+\d+\S*\@/i
describe FROM_HAS_MIXED_NUMS From: contains numbers mixed in with letters
# bug 637, more specific FROM_HAS_MIXED_NUMS, less FPs
-header FROM_HAS_MIXED_NUMS2 From =~ /\w{2,}\d{4,}[a-z]{1,2}\d{2,}\@/i
+# note: anchored for speed
+header FROM_HAS_MIXED_NUMS2 From =~ /(?:^|\W)\w{2,}\d{4,}[a-z]{1,2}\d{2,}\@/i
describe FROM_HAS_MIXED_NUMS2 From address matches known spammer format
# idea from Robert Menschel <RM...@Menschel.net>
@@ -582,7 +587,8 @@
header TO_INVESTORS To =~ /\bInvestors\@/
describe TO_INVESTORS To: non-existent 'Investors' address
-header TO_MALFORMED To !~ /(?:(?:\"[^\"]+\"|\S+)\@\S+\.\S+|^\s*.+:\s*;|^\s*\"[^\"]+\":\s*;|^\s*\([^\)]*\)\s*$|<\S+(?:\!\S+){1,}>|^\s*$)/ [if-unset: unset@unset.unset]
+# note: the first alternation is anchored for speed
+header TO_MALFORMED To !~ /(?:^|[^\S"])(?:(?:\"[^\"]+\"|\S+)\@\S+\.\S+|^\s*.+:\s*;|^\s*\"[^\"]+\":\s*;|^\s*\([^\)]*\)\s*$|<\S+(?:\!\S+){1,}>|^\s*$)/ [if-unset: unset@unset.unset]
describe TO_MALFORMED To: has a malformed address
header SUBJ_HAS_TIME_ID Subject =~ /\sTime[: ]+\d+:\d+:\d+ [AP]M\s*$/i
Modified: spamassassin/trunk/rules/20_meta_tests.cf
==============================================================================
--- spamassassin/trunk/rules/20_meta_tests.cf (original)
+++ spamassassin/trunk/rules/20_meta_tests.cf Mon Aug 2 19:47:32 2004
@@ -114,7 +114,7 @@
body __NIGERIAN_BODY_42 /nigerian? (?:national|government)/i
body __NIGERIAN_BODY_43 /\blate .{0,15}(?:father|wife|husband|general|president|daughter|son|minister).{0,20}(?:wealthy|treasure|deposit|money|left|property|known|\$|US)/i
body __NIGERIAN_BODY_44 /\b(?:of|the) late president\b/i
-body __NIGERIAN_BODY_45 /\bsum of ?(?:million|US|\$)|(?:deposit|left|huge|discovered|abandoned).{0,15} sum of/i
+body __NIGERIAN_BODY_45 /\bsum of ?(?:million|US|\$)|\b(?:deposit|left|huge|discovered|abandoned).{0,15} sum of/i
body __NIGERIAN_BODY_46 /\bgive\s+you .{0,15}(?:fund|money|total|sum|contact|percent)\b/i
##############################################################################
Modified: spamassassin/trunk/rules/50_scores.cf
==============================================================================
--- spamassassin/trunk/rules/50_scores.cf (original)
+++ spamassassin/trunk/rules/50_scores.cf Mon Aug 2 19:47:32 2004
@@ -760,12 +760,15 @@
# End of generated scores.
-# Scores for tests that are scored manually or with isolated rescore runs. Most
-# are net tests, userconf tests, tests occuring with very low frequency, or
-# tests with many false positives.
+# Scores for tests that are scored manually or with isolated rescore runs.
+# Most are net tests, userconf tests, tests occuring with very low frequency,
+# or tests with many false positives.
-# GTUBE - Generic Test for Unsollicited Bulk Email
+# GTUBE - Generic Test for Unsolicited Bulk Email
score GTUBE 1000.000
+
+# long header test
+score HEAD_LONG 2.5
# HTML control test
score HTML_MESSAGE 0.009 0.001 0.010 0.005