You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by qu...@apache.org on 2004/08/03 04:47:32 UTC

svn commit: rev 35614 - in spamassassin/trunk: lib/Mail/SpamAssassin lib/Mail/SpamAssassin/Message rules

Author: quinlan
Date: Mon Aug  2 19:47:32 2004
New Revision: 35614

Modified:
   spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
   spamassassin/trunk/rules/20_head_tests.cf
   spamassassin/trunk/rules/20_meta_tests.cf
   spamassassin/trunk/rules/50_scores.cf
Log:
various performance improvements, long header test


Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Bayes.pm	Mon Aug  2 19:47:32 2004
@@ -69,13 +69,12 @@
 # *less* these well-known headers; that way we can pick up spammers' tracking
 # headers (which are obviously not well-known in advance!).
 
+# Received is handled specially
 $IGNORED_HDRS = qr{(?: (?:X-)?Sender    # misc noise
   |Delivered-To |Delivery-Date
   |(?:X-)?Envelope-To
   |X-MIME-Auto[Cc]onverted |X-Converted-To-Plain-Text
 
-  |Received     # handled specially
-
   |Subject      # not worth a tiny gain vs. to db size increase
 
   # Date: can provide invalid cues if your spam corpus is
@@ -490,39 +489,43 @@
 sub tokenize_headers {
   my ($self, $msg) = @_;
 
-  my $hdrs = $msg->get_all_headers() . $msg->get_all_metadata();
-
   my %parsed = ();
 
-  # we don't care about whitespace; so fix continuation lines to make the next
-  # bit easier
-  $hdrs =~ s/\n[ \t]+/ /gs;
-
-  # first, keep a copy of Received hdrs, so we can strip down to last 2
-  my @rcvdlines = ($hdrs =~ /^Received: [^\n]*$/gim);
-
-  # and now delete lines for headers we don't want (incl all Receiveds)
-  $hdrs =~ s/^${IGNORED_HDRS}: [^\n]*$//gim;
+  my %user_ignore;
+  $user_ignore{$_} = 1 for @{$self->{main}->{conf}->{bayes_ignore_headers}};
 
-  if (IGNORE_MSGID_TOKENS) { $hdrs =~ s/^Message-I[dD]: [^\n]*$//gim;}
+  # get headers in array context
+  my @hdrs;
+  my @rcvdlines;
+  for ($msg->get_all_headers()) {
+    # first, keep a copy of Received headers, so we can strip down to last 2
+    if (/^Received:/i) {
+      push(@rcvdlines, $_);
+      next;
+    }
+    # and now skip lines for headers we don't want (including all Received)
+    next if /^${IGNORED_HDRS}:/i;
+    next if IGNORE_MSGID_TOKENS && /^Message-ID:/i;
+    push(@hdrs, $_);
+  }
+  push(@hdrs, $msg->get_all_metadata());
 
   # and re-add the last 2 received lines: usually a good source of
   # spamware tokens and HELO names.
-  if ($#rcvdlines >= 0) { $hdrs .= "\n".$rcvdlines[$#rcvdlines]; }
-  if ($#rcvdlines >= 1) { $hdrs .= "\n".$rcvdlines[$#rcvdlines-1]; }
-
-  # remove user-specified headers here, after Received, in case they
-  # want to ignore that too
-  foreach my $conf (@{$self->{main}->{conf}->{bayes_ignore_headers}}) {
-    $hdrs =~ s/^\Q${conf}\E: [^\n]*$//gim;
-  }
+  if ($#rcvdlines >= 0) { push(@hdrs, $rcvdlines[$#rcvdlines]); }
+  if ($#rcvdlines >= 1) { push(@hdrs, $rcvdlines[$#rcvdlines-1]); }
 
-  while ($hdrs =~ /^(\S+): ([^\n]*)$/gim) {
-    my $hdr = $1;
-    my $val = $2;
+  for (@hdrs) {
+    next unless /\S/;
+    my ($hdr, $val) = split(/:/, $_, 2);
+    $val ||= '';
+
+    # remove user-specified headers here, after Received, in case they
+    # want to ignore that too
+    next if exists $user_ignore{$hdr};
 
     # special tokenization for some headers:
-    if ($hdr =~ /^(?:|X-|Resent-)Message-I[dD]$/) {
+    if ($hdr =~ /^(?:|X-|Resent-)Message-Id$/i) {
       $val = $self->pre_chew_message_id ($val);
     }
     elsif (PRE_CHEW_ADDR_HEADERS && $hdr =~ /^(?:|X-|Resent-)
@@ -544,8 +547,7 @@
     }
 
     if (MAP_HEADERS_MID) {
-      if ($hdr eq 'In-Reply-To' || $hdr eq 'References' || $hdr =~ /^Message-Id/i)
-      {
+      if ($hdr =~ /^(?:In-Reply-To|References|Message-ID)$/i) {
         $parsed{"*MI"} = $val;
       }
     }
@@ -555,7 +557,7 @@
       }
     }
     if (MAP_HEADERS_USERAGENT) {
-      if ($hdr =~ /^(?:X-Mailer|User-Agent)$/) {
+      if ($hdr =~ /^(?:X-Mailer|User-Agent)$/i) {
         $parsed{"*UA"} = $val;
       }
     }

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm	Mon Aug  2 19:47:32 2004
@@ -35,6 +35,8 @@
 # These are generic constants that may be used across several modules
 @SA_VARS = qw(
 	META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY MBX_SEPARATOR
+	MAX_BODY_LINE_LENGTH MAX_HEADER_KEY_LENGTH MAX_HEADER_VALUE_LENGTH
+	MAX_HEADER_LENGTH
 );
 
 %EXPORT_TAGS = (
@@ -175,5 +177,17 @@
 # $2 = size of message in bytes (int)
 # $3 = message status - binary (hex)
 # $4 = message ID (hex)
+
+# ---------------------------------------------------------------------------
+# values used for internal message representations
+
+# maximum byte length of lines in the body
+use constant MAX_BODY_LINE_LENGTH => 2048;
+# maximum byte length of a header key
+use constant MAX_HEADER_KEY_LENGTH => 256;
+# maximum byte length of a header value including continued lines
+use constant MAX_HEADER_VALUE_LENGTH => 8192;
+# maximum byte length of entire header
+use constant MAX_HEADER_LENGTH => 65536;
 
 1;

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm	Mon Aug  2 19:47:32 2004
@@ -184,23 +184,22 @@
 sub _check_recipients {
   my ($self) = @_;
 
-  my @address;
+  my @inputs;
 
   # ToCc: pseudo-header works best, but sometimes Bcc: is better
   for ('ToCc', 'Bcc') {
     my $to = $self->get($_);	# get recipients
     $to =~ s/\(.*?\)//g;	# strip out the (comments)
-    @address = ($to =~ m/([\w.=-]+\@\w+(?:[\w.-]+\.)+\w+)/g);
-    last if scalar(@address) >= TOCC_SIMILAR_COUNT;
+    @inputs = ($to =~ m/([\w.=-]+\@\w+(?:[\w.-]+\.)+\w+)/g);
+    last if scalar(@inputs) >= TOCC_SIMILAR_COUNT;
   }
 
   # remove duplicate addresses only when they appear next to each other
-  my @tmp = @address;
+  my @address;
   my $previous = '';
-  @address = ();
-  foreach my $current (@tmp) {
-    next if (lc($current) eq lc($previous));
-    push(@address, ($previous = $current));
+  while (my $current = shift @inputs) {
+    push(@address, ($previous = $current)) if lc($current) ne lc($previous);
+    last if @address == 256;
   }
 
   # ideas that had both poor S/O ratios and poor hit rates:
@@ -2662,22 +2661,18 @@
 }
 
 sub check_pyzor {
-  my ($self, $fulltext) = @_;
+  my ($self, $full) = @_;
 
   return 0 unless ($self->is_pyzor_available());
   return 0 if ($self->{already_checked_pyzor});
 
   $self->{already_checked_pyzor} = 1;
 
-  # note: we don't use $fulltext. instead we get the raw message,
-  # unfiltered, for pyzor to check.  ($fulltext removes MIME
-  # parts etc.)
-  my $full = $self->{msg}->get_pristine();
-  return $self->pyzor_lookup (\$full);
+  return $self->pyzor_lookup($full);
 }
 
 sub check_dcc {
-  my ($self, $fulltext) = @_;
+  my ($self, $full) = @_;
   my $have_dccifd = $self->is_dccifd_available();
 
   return 0 unless ($have_dccifd || $self->is_dcc_available() );
@@ -2691,14 +2686,11 @@
   $_ = $self->get('X-DCC-(?:[^:]+-)?Metrics');
   return 1 if /bulk/;
   
-  # note: we don't use $fulltext. instead we get the raw message,
-  # unfiltered, for DCC to check.  ($fulltext removes MIME
-  # parts etc.)
-  my $full = $self->{msg}->get_pristine();
   if ($have_dccifd) {
-    return $self->dccifd_lookup (\$full);
-  } else {
-    return $self->dcc_lookup (\$full);
+    return $self->dccifd_lookup($full);
+  }
+  else {
+    return $self->dcc_lookup($full);
   }
 }
 
@@ -3387,5 +3379,11 @@
 }
 
 ###########################################################################
+
+sub check_for_long_header {
+  my ($self) = @_;
+
+  return defined $self->{msg}->{'truncated_header'};
+}
 
 1;

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm	Mon Aug  2 19:47:32 2004
@@ -54,8 +54,6 @@
 
 @ISA = qw(Mail::SpamAssassin::Message::Node);
 
-use constant MAX_BODY_LINE_LENGTH =>        2048;
-
 # ---------------------------------------------------------------------------
 
 =item new()
@@ -170,11 +168,21 @@
 
     # Ok, there's a header here, let's go ahead and add it in.
     if ($header) {
-      my ( $key, $value ) = split ( /:\s*/, $header, 2 );
+      # Yes, the /s is needed to match \n too.
+      my ($key, $value) = split (/:\s*(?=.)/s, $header, 2);
 
       # If it's not a valid header (aka: not in the form "foo: bar"), skip it.
       if (defined $value) {
-        $self->header( $key, $value );
+	# limit the length of the pairs we store
+	if (length($key) > MAX_HEADER_KEY_LENGTH) {
+	  $key = substr($key, 0, MAX_HEADER_KEY_LENGTH);
+	  $self->{'truncated_header'} = 1;
+	}
+	if (length($value) > MAX_HEADER_VALUE_LENGTH) {
+	  $value = substr($value, 0, MAX_HEADER_VALUE_LENGTH);
+	  $self->{'truncated_header'} = 1;
+	}
+        $self->header($key, $value);
       }
     }
 
@@ -481,11 +489,11 @@
     }
 
     if ($in_body) {
-      # we run into a perl bug if the lines are astronomically long (probably due
-      # to lots of regexp backtracking); so cut short any individual line over
-      # MAX_BODY_LINE_LENGTH bytes in length.  This can wreck HTML totally -- but
-      # IMHO the only reason a luser would use MAX_BODY_LINE_LENGTH-byte lines is
-      # to crash filters, anyway.
+      # we run into a perl bug if the lines are astronomically long (probably
+      # due to lots of regexp backtracking); so cut short any individual line
+      # over MAX_BODY_LINE_LENGTH bytes in length.  This can wreck HTML
+      # totally -- but IMHO the only reason a luser would use
+      # MAX_BODY_LINE_LENGTH-byte lines is to crash filters, anyway.
       while (length ($_) > MAX_BODY_LINE_LENGTH) {
         push (@{$part_array}, substr($_, 0, MAX_BODY_LINE_LENGTH)."\n");
         substr($_, 0, MAX_BODY_LINE_LENGTH) = '';
@@ -605,9 +613,9 @@
   }
 
   # whitespace handling (warning: small changes have large effects!)
-  $text =~ s/\n+\s*\n+/\f/gs;                # double newlines => form feed
-  $text =~ tr/ \t\n\r\x0b\xa0/ /s;        # whitespace => space
-  $text =~ tr/\f/\n/;                        # form feeds => newline
+  $text =~ s/\n+\s*\n+/\f/gs;		# double newlines => form feed
+  $text =~ tr/ \t\n\r\x0b\xa0/ /s;	# whitespace => space
+  $text =~ tr/\f/\n/;			# form feeds => newline
   
   # warn "JMD $text";
 
@@ -803,9 +811,9 @@
   }
   my @ret = ();
   foreach my $key (sort keys %{$self->{metadata}->{strings}}) {
-    push (@ret, $key, ": ", $self->{metadata}->{strings}->{$key}, "\n");
+    push (@ret, "$key: " . $self->{metadata}->{strings}->{$key} . "\n");
   }
-  return join ("", @ret);
+  return (wantarray ? @ret :  join('', @ret));
 }
 
 # ---------------------------------------------------------------------------

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm	Mon Aug  2 19:47:32 2004
@@ -38,6 +38,7 @@
 use bytes;
 
 use Mail::SpamAssassin;
+use Mail::SpamAssassin::Constants qw(:sa);
 use Mail::SpamAssassin::HTML;
 use MIME::Base64;
 use MIME::QuotedPrint;
@@ -543,6 +544,8 @@
 
 =cut
 
+# TODO: this could be made much faster by only processing all headers
+# when called in array context, otherwise just do one header
 sub get_header {
   my ($self, $hdr, $raw) = @_;
   $raw ||= 0;
@@ -585,27 +588,46 @@
 
 =cut
 
+# build it and it will not bomb
 sub get_all_headers {
   my ($self, $raw, $include_mbox) = @_;
   $raw ||= 0;
   $include_mbox ||= 0;
 
-  my %cache = ();
   my @lines = ();
 
-  # we're guaranteed that get_header() will return a non-undef value here,
-  # so don't bother trying to deal with that possibility.
-  foreach ( @{$self->{header_order}} ) {
-    push(@lines, "$_: ".($self->get_header($_,$raw))[$cache{$_}++]);
+  # precalculate destination positions based on order of appearance
+  my $i = 0;
+  my %locations;
+  for my $k (@{$self->{header_order}}) {
+    push(@{$locations{lc($k)}}, $i++);
+  }
+
+  # process headers in order of first appearance
+  my $header;
+  my $size = 0;
+  HEADER: for my $name (sort { $locations{$a}->[0] <=> $locations{$b}->[0] }
+			keys %locations)
+  {
+    # get all same-name headers and poke into correct position
+    my $positions = $locations{$name};
+    for my $contents ($self->get_header($name, $raw)) {
+      my $position = shift @{$positions};
+      $size += length($name) + length($contents) + 2;
+      if ($size > MAX_HEADER_LENGTH) {
+	$self->{'truncated_header'} = 1;
+	last HEADER;
+      }
+      $lines[$position] = $self->{header_order}->[$position] . ": $contents";
+    }
   }
 
+  # skip undefined lines if we truncated
+  @lines = grep { defined $_ } @lines if $self->{'truncated_header'};
+
   splice @lines, 0, 0, $self->{mbox_sep} if ( $include_mbox && exists $self->{mbox_sep} );
 
-  if (wantarray) {
-    return @lines;
-  } else {
-    return join ('', @lines);
-  }
+  return wantarray ? @lines : join ('', @lines);
 }
 
 # ---------------------------------------------------------------------------

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	(original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	Mon Aug  2 19:47:32 2004
@@ -153,8 +153,7 @@
 
     my $bodytext = $self->get_decoded_body_text_array();
 
-    my $fulltext = join ('', $self->{msg}->get_all_headers(1), "\n",
-			 $self->{msg}->get_pristine_body());
+    my $fulltext = $self->{msg}->get_pristine();
 
     # use $bodytext here because $decoded is too stripped
     # TVD: leave it up to get_uri_list to do the right thing ...

Modified: spamassassin/trunk/rules/20_head_tests.cf
==============================================================================
--- spamassassin/trunk/rules/20_head_tests.cf	(original)
+++ spamassassin/trunk/rules/20_head_tests.cf	Mon Aug  2 19:47:32 2004
@@ -24,6 +24,9 @@
 
 require_version @@VERSION@@
 
+header HEAD_LONG		eval:check_for_long_header()
+describe HEAD_LONG		Message headers are very long
+
 header NO_REAL_NAME		From =~ /^["\s]*\<?\S+\@\S+\>?\s*$/
 describe NO_REAL_NAME		From: does not include a real name
 
@@ -33,11 +36,13 @@
 header FROM_STARTS_WITH_NUMS	From =~ /^\d\d/
 describe FROM_STARTS_WITH_NUMS	From: starts with nums
 
-header FROM_HAS_MIXED_NUMS	From =~ /\d+[a-z]+\d+\S*\@/i
+# note: anchored for speed
+header FROM_HAS_MIXED_NUMS	From =~ /(?:^|\D)\d+[a-z]+\d+\S*\@/i
 describe FROM_HAS_MIXED_NUMS	From: contains numbers mixed in with letters
 
 # bug 637, more specific FROM_HAS_MIXED_NUMS, less FPs
-header FROM_HAS_MIXED_NUMS2	From =~ /\w{2,}\d{4,}[a-z]{1,2}\d{2,}\@/i
+# note: anchored for speed
+header FROM_HAS_MIXED_NUMS2	From =~ /(?:^|\W)\w{2,}\d{4,}[a-z]{1,2}\d{2,}\@/i
 describe FROM_HAS_MIXED_NUMS2	From address matches known spammer format
 
 # idea from Robert Menschel <RM...@Menschel.net>
@@ -582,7 +587,8 @@
 header TO_INVESTORS             To =~ /\bInvestors\@/
 describe TO_INVESTORS           To: non-existent 'Investors' address
 
-header TO_MALFORMED             To !~ /(?:(?:\"[^\"]+\"|\S+)\@\S+\.\S+|^\s*.+:\s*;|^\s*\"[^\"]+\":\s*;|^\s*\([^\)]*\)\s*$|<\S+(?:\!\S+){1,}>|^\s*$)/ [if-unset: unset@unset.unset]
+# note: the first alternation is anchored for speed
+header TO_MALFORMED             To !~ /(?:^|[^\S"])(?:(?:\"[^\"]+\"|\S+)\@\S+\.\S+|^\s*.+:\s*;|^\s*\"[^\"]+\":\s*;|^\s*\([^\)]*\)\s*$|<\S+(?:\!\S+){1,}>|^\s*$)/ [if-unset: unset@unset.unset]
 describe TO_MALFORMED           To: has a malformed address
 
 header SUBJ_HAS_TIME_ID         Subject =~ /\sTime[: ]+\d+:\d+:\d+ [AP]M\s*$/i

Modified: spamassassin/trunk/rules/20_meta_tests.cf
==============================================================================
--- spamassassin/trunk/rules/20_meta_tests.cf	(original)
+++ spamassassin/trunk/rules/20_meta_tests.cf	Mon Aug  2 19:47:32 2004
@@ -114,7 +114,7 @@
 body __NIGERIAN_BODY_42		/nigerian? (?:national|government)/i
 body __NIGERIAN_BODY_43         /\blate .{0,15}(?:father|wife|husband|general|president|daughter|son|minister).{0,20}(?:wealthy|treasure|deposit|money|left|property|known|\$|US)/i
 body __NIGERIAN_BODY_44         /\b(?:of|the) late president\b/i
-body __NIGERIAN_BODY_45         /\bsum of ?(?:million|US|\$)|(?:deposit|left|huge|discovered|abandoned).{0,15} sum of/i
+body __NIGERIAN_BODY_45         /\bsum of ?(?:million|US|\$)|\b(?:deposit|left|huge|discovered|abandoned).{0,15} sum of/i
 body __NIGERIAN_BODY_46         /\bgive\s+you .{0,15}(?:fund|money|total|sum|contact|percent)\b/i
 
 ##############################################################################

Modified: spamassassin/trunk/rules/50_scores.cf
==============================================================================
--- spamassassin/trunk/rules/50_scores.cf	(original)
+++ spamassassin/trunk/rules/50_scores.cf	Mon Aug  2 19:47:32 2004
@@ -760,12 +760,15 @@
 
 # End of generated scores.
 
-# Scores for tests that are scored manually or with isolated rescore runs.  Most
-# are net tests, userconf tests, tests occuring with very low frequency, or
-# tests with many false positives.
+# Scores for tests that are scored manually or with isolated rescore runs.
+# Most are net tests, userconf tests, tests occuring with very low frequency,
+# or tests with many false positives.
 
-# GTUBE - Generic Test for Unsollicited Bulk Email
+# GTUBE - Generic Test for Unsolicited Bulk Email
 score GTUBE 1000.000
+
+# long header test
+score HEAD_LONG 2.5
 
 # HTML control test
 score HTML_MESSAGE 0.009 0.001 0.010 0.005