You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2005/08/10 06:38:19 UTC

svn commit: r231168 - /spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores

Author: jm
Date: Tue Aug  9 21:38:18 2005
New Revision: 231168

URL: http://svn.apache.org/viewcvs?rev=231168&view=rev
Log:
attempt to support multiple <gen:mutable> blocks safely, since some of the later ones may be in ifplugin scopes

Modified:
    spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores

Modified: spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores
URL: http://svn.apache.org/viewcvs/spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores?rev=231168&r1=231167&r2=231168&view=diff
==============================================================================
--- spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores (original)
+++ spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores Tue Aug  9 21:38:18 2005
@@ -20,10 +20,16 @@
 
 my $NUM_SCORESETS = 4;
 
+my $ZERO_MINISCULE_SCORES =     1;
+my $MINISCULE_THRESHOLD =       0.1;      # points
+
+my $UNZERO_META_PREDICATES =    1;
+
 # scores are broken into three regions:
 # 1. "pre" (stuff before generated mutable scores)
-# 2. "gen" (generated mutable scores)
+# 2. "gen" (first generated mutable scores section)
 # 3. "end" (stuff after generated mutable scores)
+# 4. "gen2" (any later generated mutable scores sections)
 
 # options
 my ($scoreset, $oldscores, $newscores) = @ARGV;
@@ -39,21 +45,39 @@
 my %gascores = ();		# generated scores
 
 # variables filled-out in read_oldscores()
-my $pre = '';			# stuff before "gen" scores
-my $end = '';			# stuff after "gen" scores
+my $pre = '';			# stuff before first "gen" section
+my $end = '';			# stuff after first "gen" section
 my %oldscores;			# old scores
 my %comment;			# "gen" rule comments
 my %fixed;			# scores that are fixed (non-gen)
+my %gen2;			# scores that are gen in the $end string
+
+# compiled output
+my @gen_order = ();
+my %gen_lines = ();
 
 # read stuff in
 read_rules();
 read_gascores();
 read_oldscores();
 
+build_new_scores();
+
+if ($ZERO_MINISCULE_SCORES) {
+  fixup_miniscule_scores();
+}
+if ($UNZERO_META_PREDICATES) {
+  fixup_meta_predicates();
+}
+
+$end = sub_gen2($end);
+
 # write stuff out
 print $pre;
 print_gen();
 print $end;
+exit;
+
 
 sub read_rules {
   system ("./parse-rules-for-masses -s $scoreset") and die;
@@ -101,6 +125,7 @@
 
   # state of things
   my $where = "pre";		# region of original scores file that we're in
+  my $seen_gen = 0;             # have we seen the first <gen:mutable> tag?
 
   # read everything in
   while (my $line = <IN>) {
@@ -115,13 +140,23 @@
     elsif ($where eq "gen") {
       readline_gen($line);
     }
+    elsif ($where eq "gen2") {
+      readline_gen2($line);
+      $end .= $line;
+    }
     elsif ($where eq "end") {
       readline_fix($line);
       $end .= $line;
     }
 
     if ($line =~ /<gen:mutable>/) {
-      $where = "gen";
+      if ($seen_gen) {
+        $where = "gen2";
+      }
+      else {
+        $where = "gen";
+        $seen_gen = 1;
+      }
     }
   }
 }
@@ -159,9 +194,26 @@
   }
 }
 
-sub print_gen {
-  print "\n";
+sub readline_gen2 {
+  my ($line) = @_;
+
+  my $comment;
+  if ($line =~ s/\s*#\s*(.*)//) {
+    $comment = $1;
+    $comment =~ s/ n=$scoreset//;
+  }
+  if ($line =~ /^\s*score\s+(\S+)\s/) {
+    my (undef, $name, @scores) = split(' ', $line);
+    for (my $i = 1; $i < $NUM_SCORESETS; $i++) {
+      $scores[$i] = $scores[0] unless defined $scores[$i];
+    }
+    @{$oldscores{$name}} = @scores;
+    $comment{$name} = $comment if $comment;
+    $gen2{$name}++;
+  }
+}
 
+sub build_new_scores {
   # we just consider scores for this set that are in the input or were in the
   # "gen" region from the old scores, tmp/rules.pl is not considered here
   my %gen;				# rules to be printed in "gen" region
@@ -192,10 +244,11 @@
       delete $oldscores{$name};
     }
     else {
-      # I think these are non-issues
-      if (defined $rules{$name}->{score} && !$rules{$name}->{issubrule}) {
-	#warn "$name has no GA score, but is in tmp/rules.pl\n";
-      }
+      # zero for current scoreset if there was no new score;
+      # when the perceptron does this for mutable rules, it means
+      # that score had a new score of 0
+      $scores[$scoreset] = 0;
+
       if (defined $oldscores{$name}) {
 	$comment .= " n=$scoreset";
 	#warn "$name has no GA score, but had a score before\n";
@@ -207,15 +260,49 @@
     $unique{$_} = 1 for split(' ', $comment);
     $comment = join(' ', sort keys %unique);
 
-    # create new score line
-    printf("score %s %s%s\n", $name,
-	   join(" ", generate_scores($name, @scores)),
-	   ($comment) ? ' # ' . $comment : '');
+    push (@gen_order, $name);
+    $gen_lines{$name} = {
+      scores => \@scores,
+      comment => $comment
+    };
   }
+}
+
+sub new_score_line {
+  my ($name) = @_;
 
+  # create new score line
+  my @scores = @{$gen_lines{$name}{scores}};
+  my $comment = $gen_lines{$name}{comment};
+  return sprintf("score %s %s%s", $name,
+          join(" ", generate_scores($name, @scores)),
+          ($comment) ? ' # ' . $comment : '');
+}
+
+sub print_gen {
+  print "\n";
+  foreach my $name (@gen_order) {
+    next if ($gen2{$name});       # will do that separately
+    print new_score_line($name), "\n";
+  }
   print "\n";
 }
 
+sub sub_gen2 {
+  my $end = shift;
+
+  foreach my $name (keys %gen2) {
+    if ($end !~ s/^\s*score\s+${name}\s.+?$/
+            new_score_line($name);
+        /em)
+    {
+      # we failed to sub it; output score in main gen:mutable block instead
+      delete $gen2{$name};
+    }
+  }
+  $end;
+}
+
 sub generate_scores {
   my ($name, @scores) = @_;
 
@@ -232,9 +319,6 @@
     $scores[0] ||= 0;
   }
 
-  # zero for current scoreset if there was no new score
-  $scores[$scoreset] = 0 if !$gascores{$name};
-
   my $flag = 1;
   for (my $i = 1; $i < $NUM_SCORESETS; $i++) {
     $scores[$i] = $scores[0] unless defined $scores[$i];
@@ -259,4 +343,62 @@
   }
 
   return @scores;
+}
+
+sub fixup_miniscule_scores {
+  my $num_fixed = 0;
+
+  foreach my $name (@gen_order) {
+    my @scores = @{$gen_lines{$name}{scores}};
+    if (abs($scores[$scoreset]) < $MINISCULE_THRESHOLD) {
+      $scores[$scoreset] = 0;
+      $num_fixed++;
+    }
+    @{$gen_lines{$name}{scores}} = @scores;
+  }
+
+  warn "zeroed $num_fixed scores for being 'miniscule'.\n";
+}
+
+sub fixup_meta_predicates {
+  # this is the opposite of t/meta.t
+
+  while (my ($name, $info) = each %rules)
+  {
+    my $type = $info->{type} || "unknown";
+    # look at meta rules that are not disabled
+    next unless ($type eq "meta" && ($name =~ /^__/ || $info->{score} != 0));
+
+    next unless ($info->{depends});
+
+    # test rules should not impose requirements on release rules; ignore
+    # any dependency requirements caused by T_ rules
+    next if $name =~ /^T_/;
+
+    for my $depend (@{ $info->{depends} }) {
+      if (!exists $rules{$depend}) {
+        warn "$name depends on $depend which is nonexistent\n";
+        next;
+      }
+
+      # if dependency is a predicate, it'll run
+      next if $depend =~ /^__/;
+
+      # not a generated rule?  not our problem, then; t/meta.t will catch it
+      next unless (exists $gen_lines{$depend});
+
+      # ignore "tflags net" and "tflags learn" rules -- it is OK
+      # for those to have zero scores in some scoresets, for obvious
+      # reasons.
+      next if (defined $rules{$depend}->{tflags} &&
+              $rules{$depend}->{tflags} =~ /\b(?:net|learn)\b/);
+
+      # if dependency has a non-zero score, it'll run
+      my $depscore = $gen_lines{$depend}{scores}[$scoreset];
+      next if (defined $depscore && $depscore != 0);
+
+      warn "dep failure: $name depends on $depend with 0 score; fixing at non-0\n";
+      $gen_lines{$depend}{scores}[$scoreset] = 0.001;
+    }
+  }
 }