svn commit: r514663 - /spamassassin/trunk/masses/rule-dev/seek-phrases-in-log

[jm...@apache.org]

Author: jm
Date: Mon Mar  5 05:09:48 2007
New Revision: 514663

URL: http://svn.apache.org/viewvc?view=rev&rev=514663
Log:
optimisations; no need to do a lot of the work performed for the ham case

Modified:
    spamassassin/trunk/masses/rule-dev/seek-phrases-in-log

Modified: spamassassin/trunk/masses/rule-dev/seek-phrases-in-log
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/rule-dev/seek-phrases-in-log?view=diff&rev=514663&r1=514662&r2=514663
==============================================================================
--- spamassassin/trunk/masses/rule-dev/seek-phrases-in-log (original)
+++ spamassassin/trunk/masses/rule-dev/seek-phrases-in-log Mon Mar  5 05:09:48 2007
@@ -2,7 +2,7 @@
 
 =head1 NAME
 
-phrase-extract-in-log - extract good-looking rule phrases from a text-dump mc log
+seek-phrases-in-log - extract good-looking rules from a text-dump mc log
 
 =cut
 
@@ -50,34 +50,35 @@
 warn "reading $fs...\n";
 open IN, "<$fs" or die "cannot open spam log $fs";
 while (<IN>) {
-  /^text: (.*)$/ and proc_text(1, $1);
+  /^text: (.*)$/ and proc_text_spam($1);
 }
 close IN;
+warn "n-grams active: ".(scalar keys %ngram_count)."\n";
 
 warn "reading $fh...\n";
 open IN, "<$fh" or die "cannot open ham log $fh";
 while (<IN>) {
-  /^text: (.*)$/ and proc_text(0, $1);
+  /^text: (.*)$/ and proc_text_ham($1);
 }
 close IN;
+warn "n-grams active: ".(scalar keys %ngram_count)."\n";
 
-%word2sym = ();     # free this, no longer needed
+undef %word2sym;     # free this, no longer needed
 
 summarise();
 exit;
 
 
-sub proc_text {
-  my ($adding, $text) = @_;
+sub proc_text_spam {
+  my ($text) = @_;
 
+  # we only need to chop off the end of spam samples
   if (length($text) > $MAX_TEXT_IN_MESSAGE) {
     $text = substr $text, 0, $MAX_TEXT_IN_MESSAGE;      # chop!
   }
 
   # we only need to save spam samples in memory, ignore ham samples
-  if ($adding) {
-    push @text_string, $text;
-  }
+  push @text_string, $text;
 
   my $cp = pack "l", $msgcount;
   $msgcount++;
@@ -103,20 +104,52 @@
     # simple bayesian N-grams to start
     $tokens{"$w3.$w2.$w1"} = $tokens{"$w3.$w2"} = 1;
   }
+  
+  # deal with leftovers
+  if ($w2 && $w1) {
+    $tokens{"$w2.$w1"} = 1;
+  }
 
   foreach my $tok (keys %tokens) {
-    if ($adding) {
-      # incr the counter for this token
-      $ngram_count{$tok}++;
-      $msg_subset_hit{$tok} .= $cp;    # the message subset hit by this tok
-
-    } else {
-      # we're not tracking hits; we're killing false positives. 
-      # as soon as a single FP appears, kill all record of that token,
-      # it cannot be used
-      delete $ngram_count{$tok};
-      delete $msg_subset_hit{$tok};
+    # incr the counter for this token
+    $ngram_count{$tok}++;
+    $msg_subset_hit{$tok} .= $cp;    # the message subset hit by this tok
+  }
+}
+
+sub proc_text_ham {
+  my ($text) = @_;
+
+  my $w1 = '';
+  my $w2 = '';
+  my $w3 = '';
+
+  my %tokens = ();
+  foreach my $w (split(' ', $text)) {
+    $w3 = $w2;
+    $w2 = $w1;
+
+    $w1 = $word2sym{$w};
+    if (!$w1) {
+      # since we're deleting, there's no need to add new words
+      # to the dictionary; just use the final $sym_acc to mean
+      # "unknown ham word", and don't increment it
+      $word2sym{$w} = $w1 = $sym_acc;
     }
+
+    $tokens{"$w3.$w2.$w1"} = $tokens{"$w3.$w2"} = 1;
+  }
+  
+  if ($w2 && $w1) {
+    $tokens{"$w2.$w1"} = 1;
+  }
+
+  foreach my $tok (keys %tokens) {
+    # we're not tracking hits; we're killing false positives. 
+    # as soon as a single FP appears, kill all record of that token,
+    # it cannot be used
+    delete $ngram_count{$tok};
+    delete $msg_subset_hit{$tok};
   }
 }
 
@@ -160,9 +193,12 @@
     push @{$all_patterns_for_set{$set}}, decode_sym2words($id);
   }
 
+  warn "message subsets found: ".(scalar keys %all_patterns_for_set)."\n";
+
   my %done_set = ();
 
   printf ("%6s  %6s  %6s   %s\n", "RATIO", "SPAM%", "HAM%", "DATA");
+  $| = 1;
   foreach my $id (sort {
                       $ngram_count{$a} <=> $ngram_count{$b}
                   } keys %ngram_count)
@@ -184,12 +220,15 @@
 sub decode_sym2words {
   my $ids = shift;
   my $r;
-  if ($ids =~ /^(.*)\.(.*)\.(.*)$/) {
+  if ($ids =~ /^([^.]*)\.([^.]*)\.([^.]*)$/) {
     $r = "$sym2word{$1} $sym2word{$2} $sym2word{$3}";
   }
-  elsif ($ids =~ /^(.*)\.(.*)$/) {
+  elsif ($ids =~ /^([^.]*)\.([^.]*)$/) {
     $r = "$sym2word{$1} $sym2word{$2}";
   }
+  else {
+    warn "bad decode_sym2words format: '$ids'";
+  }
   $r =~ s/^\s+//;
   return $r;
 }
@@ -217,7 +256,7 @@
     }
 
     # we don't have all day!
-    my $pat_maxlen = 32768;
+    my $pat_maxlen = $MAX_TEXT_IN_MESSAGE;
     my $s = $hits[0];
 
     # Now, expand the pattern using a BLAST-style algorithm
@@ -228,11 +267,7 @@
       last if ($l > $pat_maxlen);     # too long
 
       my $found;
-      # this is too slow
       if ($s =~ /(.)\Q$pat\E/s) { $found = $1; }
-
-      # this is faster, since it doesn't start with a (.)
-      # if ($s =~ /\Q$pat\E/s) { $found = substr($s, pos($s) - $l, 1); }
 
       if (!defined $found) {
         # start of string.  break