You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2009/08/17 22:56:52 UTC
svn commit: r805137 - in /spamassassin/trunk: build/ masses/
Author: jm
Date: Mon Aug 17 20:56:52 2009
New Revision: 805137
URL: http://svn.apache.org/viewvc?rev=805137&view=rev
Log:
bug 6155: garescorer updates for 3.3.0. use more epochs in GA runs for better scores. clarify some mass-check warning output, ensure rule name always appears at start of line. nowadays we run the GA on Solaris, so update makefile. If a rule had no default/existing score in 50_scores.cf, don't tell the GA that 1.0 is an appropriate default value; instead pick the midway point of its score range. this produces better results. remove some dead code from masses/score-ranges-from-freqs
Modified:
spamassassin/trunk/build/parse-rules-for-masses
spamassassin/trunk/masses/Makefile
spamassassin/trunk/masses/config
spamassassin/trunk/masses/config.set0
spamassassin/trunk/masses/config.set1
spamassassin/trunk/masses/config.set2
spamassassin/trunk/masses/config.set3
spamassassin/trunk/masses/logs-to-c
spamassassin/trunk/masses/rewrite-cf-with-new-scores
spamassassin/trunk/masses/score-ranges-from-freqs
Modified: spamassassin/trunk/build/parse-rules-for-masses
URL: http://svn.apache.org/viewvc/spamassassin/trunk/build/parse-rules-for-masses?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/build/parse-rules-for-masses (original)
+++ spamassassin/trunk/build/parse-rules-for-masses Mon Aug 17 20:56:52 2009
@@ -169,6 +169,7 @@
} else {
$rules->{$rule}->{score} = $def;
}
+ $rules->{$rule}->{no_score_found} = 1;
}
# ignore net rules in set 0 or set 2
Modified: spamassassin/trunk/masses/Makefile
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/Makefile?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/Makefile (original)
+++ spamassassin/trunk/masses/Makefile Mon Aug 17 20:56:52 2009
@@ -3,8 +3,8 @@
LDFLAGS= -lm
PGAPACK= /local/pgapack-1.0.0.1
-PGAPACKLIBDIR= $(PGAPACK)/lib/linux # linux
-# PGAPACKLIBDIR= $(PGAPACK)/lib/sun4 # solaris
+#PGAPACKLIBDIR= $(PGAPACK)/lib/linux # linux
+PGAPACKLIBDIR= $(PGAPACK)/lib/sun4 # solaris
# What rule scoreset are we using?
include config
Modified: spamassassin/trunk/masses/config
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/config?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/config (original)
+++ spamassassin/trunk/masses/config Mon Aug 17 20:56:52 2009
@@ -1,5 +1,5 @@
-SCORESET=3
-HAM_PREFERENCE=5.0
+SCORESET=0
+HAM_PREFERENCE=2.0
THRESHOLD=5.0
-EPOCHS=100
+EPOCHS=500
NOTE=
Modified: spamassassin/trunk/masses/config.set0
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/config.set0?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/config.set0 (original)
+++ spamassassin/trunk/masses/config.set0 Mon Aug 17 20:56:52 2009
@@ -1,5 +1,5 @@
SCORESET=0
HAM_PREFERENCE=2.0
THRESHOLD=5.0
-EPOCHS=100
+EPOCHS=500
NOTE=
Modified: spamassassin/trunk/masses/config.set1
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/config.set1?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/config.set1 (original)
+++ spamassassin/trunk/masses/config.set1 Mon Aug 17 20:56:52 2009
@@ -1,5 +1,5 @@
SCORESET=1
HAM_PREFERENCE=5.0
THRESHOLD=5.0
-EPOCHS=100
+EPOCHS=500
NOTE=
Modified: spamassassin/trunk/masses/config.set2
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/config.set2?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/config.set2 (original)
+++ spamassassin/trunk/masses/config.set2 Mon Aug 17 20:56:52 2009
@@ -1,5 +1,5 @@
SCORESET=2
-HAM_PREFERENCE=5.0
+HAM_PREFERENCE=2.0
THRESHOLD=5.0
-EPOCHS=100
+EPOCHS=500
NOTE=
Modified: spamassassin/trunk/masses/config.set3
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/config.set3?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/config.set3 (original)
+++ spamassassin/trunk/masses/config.set3 Mon Aug 17 20:56:52 2009
@@ -1,5 +1,5 @@
SCORESET=3
HAM_PREFERENCE=5.0
THRESHOLD=5.0
-EPOCHS=100
+EPOCHS=500
NOTE=
Modified: spamassassin/trunk/masses/logs-to-c
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/logs-to-c?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/logs-to-c (original)
+++ spamassassin/trunk/masses/logs-to-c Mon Aug 17 20:56:52 2009
@@ -220,6 +220,12 @@
}
#$range_lo{$name} ||= 0.1;
#$range_hi{$name} ||= 1.5;
+
+ # no default score found? set it to max and let GA adjust downwards. this
+ # seems to help avoid a load of really good rules getting 1.0 scores
+ if ($allrules{$name}->{no_score_found}) {
+ $scores{$name} = ($range_hi{$name} + $range_lo{$name}) / 2.0;
+ }
}
$output .= ".".$i."\n".
@@ -366,7 +372,7 @@
sub read_ranges {
if (!-f 'tmp/ranges.data') {
- system ("make tmp/ranges.data SCORESET=$opt_scoreset");
+ die "need to make 'tmp/ranges.data' first";
}
# read ranges, and mutableness, from ranges.data.
@@ -382,13 +388,13 @@
my $mut = $3+0;
if ($allrules{$t}->{issubrule}) {
- # warn "ignoring '$t': is sub-rule\n"; # no need to warn
+ # warn "$t: ignoring, is sub-rule\n"; # no need to warn
$ignored_rule{$t} = 1;
$mutable_tests{$t} = 0;
next;
}
if ($t =~ /^T_/) {
- # warn "ignoring '$t': is T_ test rule\n"; # no need to warn
+ # warn "$t: ignoring, is T_ test rule\n"; # no need to warn
$ignored_rule{$t} = 1;
$mutable_tests{$t} = 0;
$range_lo{$t} = 0.01; # clamp to insignificant range
@@ -396,7 +402,7 @@
next;
}
if (($range_lo{$t} == $range_hi{$t}) && (! $range_lo{$t})) {
- warn "ignoring '$t': score and range == 0\n";
+ warn "$t: ignoring, score and range == 0\n";
$ignored_rule{$t} = 1;
$mutable_tests{$t} = 0;
next;
@@ -414,7 +420,7 @@
$mutable_tests{$t} = 1;
}
unless ($mutable_tests{$t} || $scores{$t}) {
-# warn "ignoring '$t': immutable and score == 0\n";
+ warn "$t: ignoring, immutable and score == 0 in this scoreset\n";
$ignored_rule{$t} = 1;
}
}
@@ -427,7 +433,7 @@
if ($allrules{$t}->{issubrule}) {
if (!$ignored_rule{$t}) {
- # warn "ignoring '$t': is sub-rule\n"; # no need to warn here
+ # warn "$t: ignoring, is sub-rule\n"; # no need to warn here
$ignored_rule{$t} = 1;
}
$mutable_tests{$t} = 0;
@@ -435,7 +441,7 @@
}
if ($t =~ /^T_/) {
if (!$ignored_rule{$t}) {
- # warn "ignoring '$t': is test rule\n"; # no need to warn here
+ # warn "$t: ignoring, is T_ test rule\n"; # no need to warn here
$ignored_rule{$t} = 1;
$range_lo{$t} = 0.01; # clamp to insignificant range
$range_hi{$t} = 0.01;
@@ -450,7 +456,7 @@
}
unless ($mutable_tests{$t} || $scores{$t}) {
if (!$ignored_rule{$t}) {
- warn "ignoring '$t': immutable and score == 0\n";
+ warn "$t: ignoring, immutable and score == 0 in this scoreset\n";
$ignored_rule{$t} = 1;
}
}
Modified: spamassassin/trunk/masses/rewrite-cf-with-new-scores
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/rewrite-cf-with-new-scores?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/rewrite-cf-with-new-scores (original)
+++ spamassassin/trunk/masses/rewrite-cf-with-new-scores Mon Aug 17 20:56:52 2009
@@ -514,7 +514,7 @@
my $depscore = $gen_lines{$depend}{scores}[$opt_scoreset];
next if (defined $depscore && $depscore != 0);
- warn "dep failure: $name depends on $depend with 0 score; fixing at non-0\n";
+ warn "fix meta dep: $name depends on $depend with 0 score, fixing at non-0\n";
$gen_lines{$depend}{scores}[$opt_scoreset] = 0.001;
}
}
Modified: spamassassin/trunk/masses/score-ranges-from-freqs
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/score-ranges-from-freqs?rev=805137&r1=805136&r2=805137&view=diff
==============================================================================
--- spamassassin/trunk/masses/score-ranges-from-freqs (original)
+++ spamassassin/trunk/masses/score-ranges-from-freqs Mon Aug 17 20:56:52 2009
@@ -17,46 +17,10 @@
# limitations under the License.
# </...@LICENSE>
-# NOTE: I don't think these window algorithms are used anymore! In
-# fact I think it's just [ 0 .. (4.5 * ranking) ].
-
-# (rough) graphic demo of this algorithm:
-# 0.0 = -limit [......] 0 ........ limit
-# 0.25 = -limit ..[..... 0 .]...... limit
-# 0.5 = -limit ....[... 0 ...].... limit
-# 0.75 = -limit ......[. 0 .....].. limit
-# 1.0 = -limit ........ 0 [......] limit
-my $sliding_window_limits = 4.8; # limits = [-$range, +$range]
-my $sliding_window_size = 5.5; # scores have this range within limits
-
-# 0.0 = -limit [......] 0 ........ limit
-# 0.25 = -limit ....[... 0 ]....... limit
-# 0.5 = -limit ......[. 0 .]...... limit (note: tighter)
-# 0.75 = -limit .......[ 0 ...].... limit
-# 1.0 = -limit ........ 0 [......] limit
-my $shrinking_window_lower_base = 0.00;
-my $shrinking_window_lower_range = 1.00; # *ratio, added to above
-my $shrinking_window_size_base = 1.00;
-my $shrinking_window_size_range = 1.00; # *ratio, added to above
-
-my $use_sliding_window = 0;
-
my $argcffile = shift @ARGV;
my $scoreset = shift @ARGV;
$scoreset = 0 if ( !defined $scoreset );
-if (defined ($argcffile) && $argcffile eq '-test') {
- # use this to debug the ranking -> score-range mapping:
- for $rat (0.0, 0.25, 0.5, 0.75, 1.0) {
- my ($lo, $hi); if ($use_sliding_window) {
- ($lo, $hi) = sliding_window_ratio_to_range($rat);
- } else {
- ($lo, $hi) = shrinking_window_ratio_to_range($rat);
- }
- warn "test: $rat => [ $lo $hi ]\n";
- } exit;
-}
-
my %freq_spam = ();
my %freq_nonspam = ();
@@ -96,7 +60,7 @@
next if ($test eq '(all messages as %)');
if (!defined ($rules{$test})) {
- warn "rule $test no longer exists; ignoring\n";
+ warn "$test: rule no longer exists; ignoring\n";
next;
}
@@ -111,20 +75,20 @@
# "userconf" rules, or "net" rules in set 0/2, or "learn" rules
# in set 1/3, are nonmutable.
if ($tflags =~ /\buserconf\b/) {
- print "rule $test: immutable due to 'userconf'\n";
+ print "$test: immutable due to 'userconf'\n";
$mutable_tests{$test} = 0;
}
elsif ( ($scoreset & 1) == 0 && $tflags =~ /\bnet\b/ ) {
- print "rule $test: immutable due to 'net'\n";
+ print "$test: immutable due to 'net'\n";
$mutable_tests{$test} = 0;
}
elsif ( ($scoreset & 2) == 0 && $tflags =~ /\blearn\b/ ) {
- print "rule $test: immutable due to 'learn'\n";
+ print "$test: immutable due to 'learn'\n";
$mutable_tests{$test} = 0;
}
elsif (!$rules{$test}->{mutable}) {
# rules read from the non-mutable section
- print "rule $test: immutable according to parse-rules\n";
+ print "$test: immutable according to parse-rules\n";
$mutable_tests{$test} = 0;
}
elsif ($rules{$test}->{score} == 0) {
@@ -132,7 +96,7 @@
# to "simplify" scores down to 0. comment, since real zero-scored rules
# that were scored zero when the mass-check ran, will also have no hits
# and the 'less than 0.01%' case below takes care of that.
- # print "rule $test: immutable since score is 0\n";
+ # print "$test: immutable since score is 0\n";
# $mutable_tests{$test} = 0;
}
@@ -144,7 +108,7 @@
# less than 0.01% of messages were hit: force these rules to 0.0
if ($overall < 0.01) {
- print "rule $test: immutable and zero due to low hitrate\n";
+ print "$test: zeroing rule and marking immutable, due to low hitrate\n";
$mutable_tests{$test} = 0;
$soratio{$test} = 0.5;
$ranking{$test} = 0.0;
@@ -163,7 +127,7 @@
open (OUT, ">tmp/ranges.data");
foreach my $test (sort { $ranking{$b} <=> $ranking{$a} } keys %freq) {
if (!defined ($rules{$test})) {
- warn "no rule $test";
+ warn "$test: rule not found! forcing score to 0";
print OUT ("0 0 0 $test\n");
next;
}
@@ -198,37 +162,4 @@
close OUT;
exit;
-sub sliding_window_ratio_to_range {
- my $ratio = shift;
- my $lo = -$sliding_window_limits + ($sliding_window_size * $ratio);
- my $hi = +$sliding_window_limits - ($sliding_window_size * (1-$ratio));
- if ($lo > $hi) { # ???
- ($lo,$hi) = ($hi,$lo);
- }
- ($lo, $hi);
-}
-
-sub shrinking_window_ratio_to_range {
- my $ratio = shift;
- my $is_nice = 0;
- my $adjusted = ($ratio -.5) * 2; # adj [0,1] to [-1,1]
- if ($adjusted < 0) { $is_nice = 1; $adjusted = -$adjusted; }
-
-#$adjusted /= 1.5 if ( $ratio < 0.95 && $ratio > 0.15 ); # tvd
-
- my $lower = $shrinking_window_lower_base
- + ($shrinking_window_lower_range * $adjusted);
- my $range = $shrinking_window_size_base
- + ($shrinking_window_size_range * $adjusted);
- my $lo = $lower;
- my $hi = $lower + $range;
- if ($is_nice) {
- my $tmp = $hi; $hi = -$lo; $lo = -$tmp;
- }
- if ($lo > $hi) { # ???
- ($lo,$hi) = ($hi,$lo);
- }
-
- ($lo, $hi);
-}