You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2005/08/10 06:38:19 UTC
svn commit: r231168 -
/spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores
Author: jm
Date: Tue Aug 9 21:38:18 2005
New Revision: 231168
URL: http://svn.apache.org/viewcvs?rev=231168&view=rev
Log:
attempt to support multiple <gen:mutable> blocks safely, since some of the later ones may be in ifplugin scopes
Modified:
spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores
Modified: spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores
URL: http://svn.apache.org/viewcvs/spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores?rev=231168&r1=231167&r2=231168&view=diff
==============================================================================
--- spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores (original)
+++ spamassassin/branches/3.1/masses/rewrite-cf-with-new-scores Tue Aug 9 21:38:18 2005
@@ -20,10 +20,16 @@
my $NUM_SCORESETS = 4;
+my $ZERO_MINISCULE_SCORES = 1;
+my $MINISCULE_THRESHOLD = 0.1; # points
+
+my $UNZERO_META_PREDICATES = 1;
+
# scores are broken into three regions:
# 1. "pre" (stuff before generated mutable scores)
-# 2. "gen" (generated mutable scores)
+# 2. "gen" (first generated mutable scores section)
# 3. "end" (stuff after generated mutable scores)
+# 4. "gen2" (any later generated mutable scores sections)
# options
my ($scoreset, $oldscores, $newscores) = @ARGV;
@@ -39,21 +45,39 @@
my %gascores = (); # generated scores
# variables filled-out in read_oldscores()
-my $pre = ''; # stuff before "gen" scores
-my $end = ''; # stuff after "gen" scores
+my $pre = ''; # stuff before first "gen" section
+my $end = ''; # stuff after first "gen" section
my %oldscores; # old scores
my %comment; # "gen" rule comments
my %fixed; # scores that are fixed (non-gen)
+my %gen2; # scores that are gen in the $end string
+
+# compiled output
+my @gen_order = ();
+my %gen_lines = ();
# read stuff in
read_rules();
read_gascores();
read_oldscores();
+build_new_scores();
+
+if ($ZERO_MINISCULE_SCORES) {
+ fixup_miniscule_scores();
+}
+if ($UNZERO_META_PREDICATES) {
+ fixup_meta_predicates();
+}
+
+$end = sub_gen2($end);
+
# write stuff out
print $pre;
print_gen();
print $end;
+exit;
+
sub read_rules {
system ("./parse-rules-for-masses -s $scoreset") and die;
@@ -101,6 +125,7 @@
# state of things
my $where = "pre"; # region of original scores file that we're in
+ my $seen_gen = 0; # have we seen the first <gen:mutable> tag?
# read everything in
while (my $line = <IN>) {
@@ -115,13 +140,23 @@
elsif ($where eq "gen") {
readline_gen($line);
}
+ elsif ($where eq "gen2") {
+ readline_gen2($line);
+ $end .= $line;
+ }
elsif ($where eq "end") {
readline_fix($line);
$end .= $line;
}
if ($line =~ /<gen:mutable>/) {
- $where = "gen";
+ if ($seen_gen) {
+ $where = "gen2";
+ }
+ else {
+ $where = "gen";
+ $seen_gen = 1;
+ }
}
}
}
@@ -159,9 +194,26 @@
}
}
-sub print_gen {
- print "\n";
+sub readline_gen2 {
+ my ($line) = @_;
+
+ my $comment;
+ if ($line =~ s/\s*#\s*(.*)//) {
+ $comment = $1;
+ $comment =~ s/ n=$scoreset//;
+ }
+ if ($line =~ /^\s*score\s+(\S+)\s/) {
+ my (undef, $name, @scores) = split(' ', $line);
+ for (my $i = 1; $i < $NUM_SCORESETS; $i++) {
+ $scores[$i] = $scores[0] unless defined $scores[$i];
+ }
+ @{$oldscores{$name}} = @scores;
+ $comment{$name} = $comment if $comment;
+ $gen2{$name}++;
+ }
+}
+sub build_new_scores {
# we just consider scores for this set that are in the input or were in the
# "gen" region from the old scores, tmp/rules.pl is not considered here
my %gen; # rules to be printed in "gen" region
@@ -192,10 +244,11 @@
delete $oldscores{$name};
}
else {
- # I think these are non-issues
- if (defined $rules{$name}->{score} && !$rules{$name}->{issubrule}) {
- #warn "$name has no GA score, but is in tmp/rules.pl\n";
- }
+ # zero for current scoreset if there was no new score;
+ # when the perceptron does this for mutable rules, it means
+ # that score had a new score of 0
+ $scores[$scoreset] = 0;
+
if (defined $oldscores{$name}) {
$comment .= " n=$scoreset";
#warn "$name has no GA score, but had a score before\n";
@@ -207,15 +260,49 @@
$unique{$_} = 1 for split(' ', $comment);
$comment = join(' ', sort keys %unique);
- # create new score line
- printf("score %s %s%s\n", $name,
- join(" ", generate_scores($name, @scores)),
- ($comment) ? ' # ' . $comment : '');
+ push (@gen_order, $name);
+ $gen_lines{$name} = {
+ scores => \@scores,
+ comment => $comment
+ };
}
+}
+
+sub new_score_line {
+ my ($name) = @_;
+ # create new score line
+ my @scores = @{$gen_lines{$name}{scores}};
+ my $comment = $gen_lines{$name}{comment};
+ return sprintf("score %s %s%s", $name,
+ join(" ", generate_scores($name, @scores)),
+ ($comment) ? ' # ' . $comment : '');
+}
+
+sub print_gen {
+ print "\n";
+ foreach my $name (@gen_order) {
+ next if ($gen2{$name}); # will do that separately
+ print new_score_line($name), "\n";
+ }
print "\n";
}
+sub sub_gen2 {
+ my $end = shift;
+
+ foreach my $name (keys %gen2) {
+ if ($end !~ s/^\s*score\s+${name}\s.+?$/
+ new_score_line($name);
+ /em)
+ {
+ # we failed to sub it; output score in main gen:mutable block instead
+ delete $gen2{$name};
+ }
+ }
+ $end;
+}
+
sub generate_scores {
my ($name, @scores) = @_;
@@ -232,9 +319,6 @@
$scores[0] ||= 0;
}
- # zero for current scoreset if there was no new score
- $scores[$scoreset] = 0 if !$gascores{$name};
-
my $flag = 1;
for (my $i = 1; $i < $NUM_SCORESETS; $i++) {
$scores[$i] = $scores[0] unless defined $scores[$i];
@@ -259,4 +343,62 @@
}
return @scores;
+}
+
+sub fixup_miniscule_scores {
+ my $num_fixed = 0;
+
+ foreach my $name (@gen_order) {
+ my @scores = @{$gen_lines{$name}{scores}};
+ if (abs($scores[$scoreset]) < $MINISCULE_THRESHOLD) {
+ $scores[$scoreset] = 0;
+ $num_fixed++;
+ }
+ @{$gen_lines{$name}{scores}} = @scores;
+ }
+
+ warn "zeroed $num_fixed scores for being 'miniscule'.\n";
+}
+
+sub fixup_meta_predicates {
+ # this is the opposite of t/meta.t
+
+ while (my ($name, $info) = each %rules)
+ {
+ my $type = $info->{type} || "unknown";
+ # look at meta rules that are not disabled
+ next unless ($type eq "meta" && ($name =~ /^__/ || $info->{score} != 0));
+
+ next unless ($info->{depends});
+
+ # test rules should not impose requirements on release rules; ignore
+ # any dependency requirements caused by T_ rules
+ next if $name =~ /^T_/;
+
+ for my $depend (@{ $info->{depends} }) {
+ if (!exists $rules{$depend}) {
+ warn "$name depends on $depend which is nonexistent\n";
+ next;
+ }
+
+ # if dependency is a predicate, it'll run
+ next if $depend =~ /^__/;
+
+ # not a generated rule? not our problem, then; t/meta.t will catch it
+ next unless (exists $gen_lines{$depend});
+
+ # ignore "tflags net" and "tflags learn" rules -- it is OK
+ # for those to have zero scores in some scoresets, for obvious
+ # reasons.
+ next if (defined $rules{$depend}->{tflags} &&
+ $rules{$depend}->{tflags} =~ /\b(?:net|learn)\b/);
+
+ # if dependency has a non-zero score, it'll run
+ my $depscore = $gen_lines{$depend}{scores}[$scoreset];
+ next if (defined $depscore && $depscore != 0);
+
+ warn "dep failure: $name depends on $depend with 0 score; fixing at non-0\n";
+ $gen_lines{$depend}{scores}[$scoreset] = 0.001;
+ }
+ }
}