You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2006/12/06 15:39:47 UTC
svn commit: r483095 - in /spamassassin/branches/jm_re2c_hacks: ./ build/
lib/Mail/SpamAssassin/ lib/Mail/SpamAssassin/Conf/
lib/Mail/SpamAssassin/Plugin/ lib/Mail/SpamAssassin/Util/ masses/
masses/rule-qa/automc/ rules/ spamd/ tools/
Author: jm
Date: Wed Dec 6 06:39:45 2006
New Revision: 483095
URL: http://svn.apache.org/viewvc?view=rev&rev=483095
Log:
merged up to 483091 using 'svn merge -r478920:483091 https://svn.apache.org/repos/asf/spamassassin/trunk'
Added:
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/VBounce.pm
- copied unchanged from r483091, spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/VBounce.pm
spamassassin/branches/jm_re2c_hacks/rules/20_vbounce.cf
- copied unchanged from r483091, spamassassin/trunk/rules/20_vbounce.cf
Removed:
spamassassin/branches/jm_re2c_hacks/rules/25_body_tests_pl.cf
Modified:
spamassassin/branches/jm_re2c_hacks/CREDITS
spamassassin/branches/jm_re2c_hacks/MANIFEST
spamassassin/branches/jm_re2c_hacks/build/cf_to_html
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/ArchiveIterator.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Conf/Parser.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Constants.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Message.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/Check.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/RelayEval.pm
spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Util/RegistrarBoundaries.pm
spamassassin/branches/jm_re2c_hacks/masses/extract-results
spamassassin/branches/jm_re2c_hacks/masses/fp-fn-statistics
spamassassin/branches/jm_re2c_hacks/masses/logs-to-c
spamassassin/branches/jm_re2c_hacks/masses/mass-check
spamassassin/branches/jm_re2c_hacks/masses/mk-baseline-results
spamassassin/branches/jm_re2c_hacks/masses/rule-qa/automc/gen_info_xml
spamassassin/branches/jm_re2c_hacks/masses/runGA
spamassassin/branches/jm_re2c_hacks/rules/20_dnsbl_tests.cf
spamassassin/branches/jm_re2c_hacks/rules/25_replace.cf
spamassassin/branches/jm_re2c_hacks/rules/30_text_de.cf
spamassassin/branches/jm_re2c_hacks/rules/30_text_nl.cf
spamassassin/branches/jm_re2c_hacks/rules/50_scores.cf
spamassassin/branches/jm_re2c_hacks/rules/v320.pre
spamassassin/branches/jm_re2c_hacks/sa-learn.raw
spamassassin/branches/jm_re2c_hacks/sa-update.raw
spamassassin/branches/jm_re2c_hacks/spamassassin.raw
spamassassin/branches/jm_re2c_hacks/spamd/spamd.raw
spamassassin/branches/jm_re2c_hacks/tools/split_corpora
Modified: spamassassin/branches/jm_re2c_hacks/CREDITS
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/CREDITS?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/CREDITS (original)
+++ spamassassin/branches/jm_re2c_hacks/CREDITS Wed Dec 6 06:39:45 2006
@@ -18,6 +18,7 @@
This list contains committers in alphabetical order (and their Amazon
wishlists).
+ - Tony Finch
- Kevin A. McGrail
- Robert Menschel
- John Gardiner Myers
Modified: spamassassin/branches/jm_re2c_hacks/MANIFEST
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/MANIFEST?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/MANIFEST (original)
+++ spamassassin/branches/jm_re2c_hacks/MANIFEST Wed Dec 6 06:39:45 2006
@@ -411,7 +411,6 @@
rules/20_porn.cf
rules/20_ratware.cf
rules/20_uri_tests.cf
-rules/25_body_tests_pl.cf
rules/30_text_de.cf
rules/30_text_fr.cf
rules/30_text_it.cf
@@ -466,3 +465,5 @@
t/spamd_kill_restart_rr.t
t/re_base_extraction.t
t/spamd_unix_and_tcp.t
+lib/Mail/SpamAssassin/Plugin/VBounce.pm
+rules/20_vbounce.cf
Modified: spamassassin/branches/jm_re2c_hacks/build/cf_to_html
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/build/cf_to_html?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/build/cf_to_html (original)
+++ spamassassin/branches/jm_re2c_hacks/build/cf_to_html Wed Dec 6 06:39:45 2006
@@ -8,7 +8,8 @@
my $tmpl = q{<tr __BGCOLOR__><td><strong>__TYPE__</strong></td>
<td><font size=-1>__LANG__</font></td>
<td><font size=-1>__DESCRIPTION__</font></td>
-<td><font size=-1>__NAME__</font></td><td>__SCORE__</td></tr>};
+<td><font size=-1>__NAME__</font></td><td>__SCORE__</td>
+<td><a href='__MORE__'>Wiki</a></td></tr>};
my %test = ();
my %type = ();
@@ -63,12 +64,16 @@
my $bgcolor = '';
if ($n % 2 == 0) { $bgcolor = ' bgcolor=#ccffcc'; }
+ $t =~ s/[^A-Za-z0-9_]/_/gs; # sanitize odd chars
+ my $more = 'http://wiki.apache.org/spamassassin/Rules/'.$t;
+
s/__BGCOLOR__/$bgcolor/gs;
s/__NAME__/$t/gs;
s/__DESCRIPTION__/$desc/gs;
s/__TYPE__/$type{$t}/gs;
s/__LANG__/$lang{$t}/gs;
s/__SCORE__/$score{$t} or 1/ges;
+ s/__MORE__/$more/gs;
print;
}
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/ArchiveIterator.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/ArchiveIterator.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/ArchiveIterator.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/ArchiveIterator.pm Wed Dec 6 06:39:45 2006
@@ -34,6 +34,7 @@
use vars qw {
$MESSAGES
$AICache
+ %class_opts
@ISA
};
@@ -47,7 +48,6 @@
my $iter = new Mail::SpamAssassin::ArchiveIterator(
{
- 'opt_n' => 1,
'opt_all' => 1,
'opt_cache' => 1,
}
@@ -93,42 +93,6 @@
Typically messages over 250k are skipped by ArchiveIterator. Use this option
to keep from skipping messages based on size.
-=item opt_n
-
-ArchiveIterator is typically used to simulate ham and spam moving through
-SpamAssassin. By default, the list of messages is sorted by received date so
-that the mails can be passed through in order. If opt_n is true, the sorting
-will not occur. This is useful if you don't care about the order of the
-messages.
-
-=item opt_head
-
-Only use the first N ham and N spam (or if the value is -N, only use the first
-N total messages regardless of class).
-
-This setting can be specified separately for ham and spam target classes.
-If multiple targets for one class are specified with different
-options, the last target's options will be used.
-
-If the value is negative, and multiple targets are specified with different
-options, the last spam target's setting will be used.
-
-=item opt_tail
-
-Only use the last N ham and N spam (or if the value is -N, only use the last
-N total messages regardless of class).
-
-If both C<opt_head> and C<opt_tail> are specified, then the C<opt_head> value
-specifies a subset of the C<opt_tail> selection to use; in other words, the
-C<opt_tail> splice is applied first.
-
-This setting can be specified separately for ham and spam target classes.
-If multiple targets for one class are specified with different
-options, the last target's options will be used.
-
-If the value is negative, and multiple targets are specified with different
-options, the last spam target's setting will be used.
-
=item opt_scanprob
Randomly select messages to scan, with a probability of N, where N ranges
@@ -211,8 +175,8 @@
bless ($self, $class);
# If any of these options are set, we need to figure out the message's
- # receive date at scan time. opt_n == 0, opt_after, opt_before
- $self->{determine_receive_date} = !$self->{opt_n} ||
+ # receive date at scan time. opt_after, opt_before, or opt_want_date
+ $self->{determine_receive_date} =
defined $self->{opt_after} || defined $self->{opt_before} ||
$self->{opt_want_date};
@@ -252,9 +216,9 @@
following format: C<class:format:raw_location>, or a hash reference containing
key-value option pairs and a 'target' key with a value in that format.
-The key-value option pairs that can be used are: opt_head, opt_tail,
-opt_scanprob, opt_after, opt_before. See the constructor method's
-documentation for more information on their effects.
+The key-value option pairs that can be used are: opt_scanprob, opt_after,
+opt_before. See the constructor method's documentation for more information
+on their effects.
run() returns 0 if there was an error (can't open a file, etc,) and 1 if there
were no errors.
@@ -273,18 +237,21 @@
C<mbox> an mbox formatted file, or C<mbx> for an mbx formatted directory.
C<detect> can also be used. This assumes C<mbox> for any file whose path
-contains the pattern C</\.mbox/i>, C<file> for STDIN and anything that is
-not a directory, or C<directory> otherwise.
+contains the pattern C</\.mbox/i>, C<file> anything that is not a
+directory, or C<directory> otherwise.
=item raw_location
-Path to file or directory. Can be "-" for STDIN. File globbing is allowed
-using the standard csh-style globbing (see C<perldoc -f glob>). C<~> at the
-front of the value will be replaced by the C<HOME> environment variable.
-Escaped whitespace is protected as well.
+Path to file or directory. File globbing is allowed using the
+standard csh-style globbing (see C<perldoc -f glob>). C<~> at the
+front of the value will be replaced by the C<HOME> environment
+variable. Escaped whitespace is protected as well.
B<NOTE:> C<~user> is not allowed.
+B<NOTE 2:> C<-> is not allowed as a raw location. To have
+ArchiveIterator deal with STDIN, generate a temp file.
+
=back
=cut
@@ -297,10 +264,22 @@
return 0;
}
+ # scan the targets and get the number and list of messages
+ $self->_scan_targets(\@targets,
+ sub {
+ my($self, $date, $class, $format, $mail) = @_;
+ push(@{$self->{$class}}, _index_pack($date, $class, $format, $mail));
+ }
+ );
+
my $messages;
+ # for ease of memory, we'll play with pointers
+ $messages = $self->{s};
+ undef $self->{s};
+ push(@{$messages}, @{$self->{h}});
+ undef $self->{h};
- # scan the targets and get the number and list of messages
- ($MESSAGES, $messages) = $self->message_array(\@targets);
+ $MESSAGES = scalar(@{$messages});
# go ahead and run through all of the messages specified
return $self->_run($messages);
@@ -310,7 +289,7 @@
my ($self, $messages) = @_;
while (my $message = shift @{$messages}) {
- my($class, undef, $date, undef, $result) = $self->run_message($message);
+ my($class, undef, $date, undef, $result) = $self->_run_message($message);
&{$self->{result_sub}}($class, $result, $date) if $result;
}
return ! $self->{access_problem};
@@ -320,26 +299,26 @@
## run_message and related functions to process a single message
-sub run_message {
+sub _run_message {
my ($self, $msg) = @_;
- my ($date, $class, $format, $mail) = index_unpack($msg);
+ my ($date, $class, $format, $mail) = _index_unpack($msg);
if ($format eq 'f') {
- return $self->run_file($class, $format, $mail, $date);
+ return $self->_run_file($class, $format, $mail, $date);
}
elsif ($format eq 'm') {
- return $self->run_mailbox($class, $format, $mail, $date);
+ return $self->_run_mailbox($class, $format, $mail, $date);
}
elsif ($format eq 'b') {
- return $self->run_mbx($class, $format, $mail, $date);
+ return $self->_run_mbx($class, $format, $mail, $date);
}
}
-sub run_file {
+sub _run_file {
my ($self, $class, $format, $where, $date) = @_;
- if (!mail_open($where)) {
+ if (!_mail_open($where)) {
$self->{access_problem} = 1;
return;
}
@@ -367,13 +346,13 @@
return($class, $format, $date, $where, &{$self->{wanted_sub}}($class, $where, $date, \@msg, $format));
}
-sub run_mailbox {
+sub _run_mailbox {
my ($self, $class, $format, $where, $date) = @_;
my ($file, $offset) = ($where =~ m/(.*)\.(\d+)$/);
my @msg;
my $header;
- if (!mail_open($file)) {
+ if (!_mail_open($file)) {
$self->{access_problem} = 1;
return;
}
@@ -402,14 +381,14 @@
return($class, $format, $date, $where, &{$self->{wanted_sub}}($class, $where, $date, \@msg, $format));
}
-sub run_mbx {
+sub _run_mbx {
my ($self, $class, $format, $where, $date) = @_;
my ($file, $offset) = ($where =~ m/(.*)\.(\d+)$/);
my @msg;
my $header;
- if (!mail_open($file)) {
+ if (!_mail_open($file)) {
$self->{access_problem} = 1;
return;
}
@@ -446,11 +425,10 @@
############################################################################
-# TODO: this needs POD since mass-check uses it?
-sub message_array {
- my ($self, $targets) = @_;
+sub _scan_targets {
+ my ($self, $targets, $bkfunc) = @_;
- my %class_opts = ();
+ %class_opts = ();
foreach my $target (@${targets}) {
if (!defined $target) {
@@ -483,6 +461,11 @@
next;
}
+ if ($rawloc eq '-') {
+ warn 'archive-iterator: raw location "-" is not supported';
+ next;
+ }
+
# use ham by default, things like "spamassassin" can't specify the type
$class = substr($class, 0, 1) || 'h';
@@ -493,9 +476,9 @@
foreach my $k (keys %opts) {
$self->{$k} = $opts{$k};
}
- $self->set_default_message_selection_opts();
+ $self->_set_default_message_selection_opts();
- my @locations = $self->fix_globs($rawloc);
+ my @locations = $self->_fix_globs($rawloc);
foreach my $location (@locations) {
my $method;
@@ -504,86 +487,41 @@
# detect the format
if (!-d $location && $location =~ /\.mbox/i) {
# filename indicates mbox
- $method = \&scan_mailbox;
+ $format = 'mbox';
}
- elsif ($location eq '-' || !(-d $location)) {
- # stdin is considered a file if not passed as mbox
- $method = \&scan_file;
+ elsif (!(-d $location)) {
+ $format = 'file';
}
else {
# it's a directory
- $method = \&scan_directory;
- }
- }
- else {
- if ($format eq "dir") {
- $method = \&scan_directory;
- }
- elsif ($format eq "file") {
- $method = \&scan_file;
- }
- elsif ($format eq "mbox") {
- $method = \&scan_mailbox;
- }
- elsif ($format eq "mbx") {
- $method = \&scan_mbx;
+ $format = 'dir';
}
}
- if(defined($method)) {
- &{$method}($self, $class, $location);
+ if ($format eq 'dir') {
+ $method = \&_scan_directory;
+ }
+ elsif ($format eq 'mbox') {
+ $method = \&_scan_mailbox;
+ }
+ elsif ($format eq 'file') {
+ $method = \&_scan_file;
+ }
+ elsif ($format eq 'mbx') {
+ $method = \&_scan_mbx;
}
else {
warn "archive-iterator: format $format unknown!";
+ next;
}
- }
- }
-
- $self->top_and_tail_messages($self->{h}, $class_opts{h});
- $self->top_and_tail_messages($self->{s}, $class_opts{s});
-
- my $messages;
- if ($self->{opt_n}) {
- # OPT_N == 1 means don't bother sorting on message receive date
-
- # for ease of memory, we'll play with pointers
- $messages = $self->{s};
- undef $self->{s};
- push(@{$messages}, @{$self->{h}});
- undef $self->{h};
- }
- else {
- # OPT_N == 0 means sort on message receive date
-
- # Sort the spam and ham groups by date
- my @s = @{$self->{s}};
- undef $self->{s};
- my @h = @{$self->{h}};
- undef $self->{h};
- # interleave ordered spam and ham
- if (@s && @h) {
- my $ratio = @s / @h;
- while (@s && @h) {
- push @{$messages}, (@s / @h > $ratio) ? (shift @s) : (shift @h);
- }
+ # call the appropriate method
+ &{$method}($self, $class, $location, $bkfunc);
}
- # push the rest onto the end
- push @{$messages}, @s, @h;
}
-
- # head or tail < 0 means crop the total list, negate the value appropriately
- if ($self->{opt_tail} < 0) {
- splice(@{$messages}, 0, $self->{opt_tail});
- }
- if ($self->{opt_head} < 0) {
- splice(@{$messages}, -$self->{opt_head});
- }
-
- return scalar(@{$messages}), $messages;
}
-sub mail_open {
+sub _mail_open {
my ($file) = @_;
my $expr;
@@ -603,64 +541,26 @@
return 1;
}
-sub set_default_message_selection_opts {
+sub _set_default_message_selection_opts {
my ($self) = @_;
- $self->{opt_head} = 0 unless (defined $self->{opt_head});
- $self->{opt_tail} = 0 unless (defined $self->{opt_tail});
$self->{opt_scanprob} = 1.0 unless (defined $self->{opt_scanprob});
$self->{opt_want_date} = 1 unless (defined $self->{opt_want_date});
$self->{opt_cache} = 0 unless (defined $self->{opt_cache});
}
-sub top_and_tail_messages {
- my ($self, $ary, $opts) = @_;
-
- foreach my $k (keys %{$opts}) {
- $self->{$k} = $opts->{$k};
- }
- $self->set_default_message_selection_opts();
-
- if ($self->{opt_n}) {
- # OPT_N == 1 means don't bother sorting on message receive date
-
- # head or tail > 0 means crop each list
- if ($self->{opt_tail} > 0) {
- splice(@{$ary}, 0, -$self->{opt_tail});
- }
- if ($self->{opt_head} > 0) {
- splice(@{$ary}, min ($self->{opt_head}, scalar @{$ary}));
- }
- }
- else {
- # OPT_N == 0 means sort on message receive date
-
- # Sort the spam and ham groups by date
- my @s = sort { $a cmp $b } @{$ary};
-
- # head or tail > 0 means crop each list
- if ($self->{opt_tail} > 0) {
- splice(@s, 0, -$self->{opt_tail});
- }
- if ($self->{opt_head} > 0) {
- splice(@s, min ($self->{opt_head}, scalar @s));
- }
-
- @{$ary} = @s;
- }
-}
-
############################################################################
-sub message_is_useful_by_date {
+sub _message_is_useful_by_date {
my ($self, $date) = @_;
- return 0 unless $date; # undef or 0 date = unusable
-
if (!$self->{opt_after} && !$self->{opt_before}) {
# Not using the feature
return 1;
}
- elsif (!$self->{opt_before}) {
+
+ return 0 unless $date; # undef or 0 date = unusable
+
+ if (!$self->{opt_before}) {
# Just care about after
return $date > $self->{opt_after};
}
@@ -673,7 +573,7 @@
# make assumptions about --before, since the file may have been "touch"ed
# since the last message was appended; but we can assume that too-old
# files cannot contain messages newer than their modtime.
-sub message_is_useful_by_file_modtime {
+sub _message_is_useful_by_file_modtime {
my ($self, $date) = @_;
# better safe than sorry, if date is undef; let other stuff catch errors
@@ -687,7 +587,7 @@
}
}
-sub scanprob_says_scan {
+sub _scanprob_says_scan {
my ($self) = @_;
if (defined $self->{opt_scanprob} && $self->{opt_scanprob} < 1.0) {
if ( int( rand( 1 / $self->{opt_scanprob} ) ) != 0 ) {
@@ -706,18 +606,18 @@
# put the date in first, big-endian packed format
# this format lets cmp easily sort by date, then class, format, and path.
-sub index_pack {
+sub _index_pack {
return pack("NAAA*", @_);
}
-sub index_unpack {
+sub _index_unpack {
return unpack("NAAA*", $_[0]);
}
############################################################################
-sub scan_directory {
- my ($self, $class, $folder) = @_;
+sub _scan_directory {
+ my ($self, $class, $folder, $bkfunc) = @_;
my @files;
@@ -741,10 +641,10 @@
return;
}
- $self->create_cache('dir', $folder);
+ $self->_create_cache('dir', $folder);
foreach my $mail (@files) {
- $self->scan_file($class, $mail);
+ $self->_scan_file($class, $mail, $bkfunc);
}
if (defined $AICache) {
@@ -752,47 +652,48 @@
}
}
-sub scan_file {
- my ($self, $class, $mail) = @_;
+sub _scan_file {
+ my ($self, $class, $mail, $bkfunc) = @_;
- $self->bump_scan_progress();
+ $self->_bump_scan_progress();
my @s = stat($mail);
- return unless $self->message_is_useful_by_file_modtime($s[9]);
+ return unless $self->_message_is_useful_by_file_modtime($s[9]);
- if (!$self->{determine_receive_date}) {
- push(@{$self->{$class}}, index_pack(AI_TIME_UNKNOWN, $class, "f", $mail));
- return;
- }
+ my $date = AI_TIME_UNKNOWN;
- my $date;
- unless (defined $AICache and $date = $AICache->check($mail)) {
- my $header;
- if (!mail_open($mail)) {
- $self->{access_problem} = 1;
- return;
- }
- while (<INPUT>) {
- last if /^\s*$/;
- $header .= $_;
- }
- close(INPUT);
- $date = Mail::SpamAssassin::Util::receive_date($header);
- if (defined $AICache) {
- $AICache->update($mail, $date);
+ if ($self->{determine_receive_date}) {
+ unless (defined $AICache and $date = $AICache->check($mail)) {
+ my $header;
+ if (!_mail_open($mail)) {
+ $self->{access_problem} = 1;
+ return;
+ }
+ while (<INPUT>) {
+ last if /^\s*$/;
+ $header .= $_;
+ }
+ close(INPUT);
+ $date = Mail::SpamAssassin::Util::receive_date($header);
+ if (defined $AICache) {
+ $AICache->update($mail, $date);
+ }
}
+
+ return if !$self->_message_is_useful_by_date($date);
+ return if !$self->_scanprob_says_scan();
}
- return if !$self->message_is_useful_by_date($date);
- return if !$self->scanprob_says_scan();
- push(@{$self->{$class}}, index_pack($date, $class, "f", $mail));
+ &{$bkfunc}($self, $date, $class, 'f', $mail);
+
+ return;
}
-sub scan_mailbox {
- my ($self, $class, $folder) = @_;
+sub _scan_mailbox {
+ my ($self, $class, $folder, $bkfunc) = @_;
my @files;
- if ($folder ne '-' && -d $folder) {
+ if (-d $folder) {
# passed a directory of mboxes
$folder =~ s/\/\s*$//; #Remove trailing slash, if there
if (!opendir(DIR, $folder)) {
@@ -813,7 +714,7 @@
}
foreach my $file (@files) {
- $self->bump_scan_progress();
+ $self->_bump_scan_progress();
if ($file =~ /\.(?:gz|bz2)$/) {
warn "archive-iterator: compressed mbox folders are not supported at this time\n";
$self->{access_problem} = 1;
@@ -821,12 +722,12 @@
}
my @s = stat($file);
- next unless $self->message_is_useful_by_file_modtime($s[9]);
+ next unless $self->_message_is_useful_by_file_modtime($s[9]);
my $info = {};
my $count;
- $self->create_cache('mbox', $file);
+ $self->_create_cache('mbox', $file);
if ($self->{opt_cache}) {
if ($count = $AICache->count()) {
@@ -835,7 +736,7 @@
}
unless ($count) {
- if (!mail_open($file)) {
+ if (!_mail_open($file)) {
$self->{access_problem} = 1;
next;
}
@@ -867,7 +768,7 @@
$where = tell INPUT;
}
if ($header) {
- $self->bump_scan_progress();
+ $self->_bump_scan_progress();
$info->{$offset} = Mail::SpamAssassin::Util::receive_date($header);
}
}
@@ -880,11 +781,11 @@
}
if ($self->{determine_receive_date}) {
- next if !$self->message_is_useful_by_date($v);
+ next if !$self->_message_is_useful_by_date($v);
}
- next if !$self->scanprob_says_scan();
+ next if !$self->_scanprob_says_scan();
- push(@{$self->{$class}}, index_pack($v, $class, "m", "$file.$k"));
+ &{$bkfunc}($self, $v, $class, 'm', "$file.$k");
}
if (defined $AICache) {
@@ -893,11 +794,11 @@
}
}
-sub scan_mbx {
- my ($self, $class, $folder) = @_;
+sub _scan_mbx {
+ my ($self, $class, $folder, $bkfunc) = @_;
my (@files, $fp);
- if ($folder ne '-' && -d $folder) {
+ if (-d $folder) {
# got passed a directory full of mbx folders.
$folder =~ s/\/\s*$//; # remove trailing slash, if there is one
if (!opendir(DIR, $folder)) {
@@ -918,7 +819,7 @@
}
foreach my $file (@files) {
- $self->bump_scan_progress();
+ $self->_bump_scan_progress();
if ($folder =~ /\.(?:gz|bz2)$/) {
warn "archive-iterator: compressed mbx folders are not supported at this time\n";
@@ -927,12 +828,12 @@
}
my @s = stat($file);
- next unless $self->message_is_useful_by_file_modtime($s[9]);
+ next unless $self->_message_is_useful_by_file_modtime($s[9]);
my $info = {};
my $count;
- $self->create_cache('mbx', $file);
+ $self->_create_cache('mbx', $file);
if ($self->{opt_cache}) {
if ($count = $AICache->count()) {
@@ -941,7 +842,7 @@
}
unless ($count) {
- if (!mail_open($file)) {
+ if (!_mail_open($file)) {
$self->{access_problem} = 1;
next;
}
@@ -969,7 +870,7 @@
$header .= $_;
}
- $self->bump_scan_progress();
+ $self->_bump_scan_progress();
$info->{$offset} = Mail::SpamAssassin::Util::receive_date($header);
# go onto the next message
@@ -988,11 +889,11 @@
}
if ($self->{determine_receive_date}) {
- next if !$self->message_is_useful_by_date($v);
+ next if !$self->_message_is_useful_by_date($v);
}
- next if !$self->scanprob_says_scan();
+ next if !$self->_scanprob_says_scan();
- push(@{$self->{$class}}, index_pack($v, $class, "b", "$file.$k"));
+ &{$bkfunc}($self, $v, $class, 'b', "$file.$k");
}
if (defined $AICache) {
@@ -1003,7 +904,7 @@
############################################################################
-sub bump_scan_progress {
+sub _bump_scan_progress {
my ($self) = @_;
if (exists $self->{scan_progress_sub}) {
return unless ($self->{scan_progress_counter}++ % 50 == 0);
@@ -1016,7 +917,7 @@
{
my $home;
- sub fix_globs {
+ sub _fix_globs {
my ($self, $path) = @_;
unless (defined $home) {
@@ -1046,11 +947,7 @@
}
}
-sub min {
- return ($_[0] < $_[1] ? $_[0] : $_[1]);
-}
-
-sub create_cache {
+sub _create_cache {
my ($self, $type, $path) = @_;
if ($self->{opt_cache}) {
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Conf/Parser.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Conf/Parser.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Conf/Parser.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Conf/Parser.pm Wed Dec 6 06:39:45 2006
@@ -455,7 +455,7 @@
my $eval = '';
my $bad = 0;
foreach my $token (@tokens) {
- if ($token =~ /^(\W+|[\-\+\d\.]+)$/) {
+ if ($token =~ /^(\W+|[+-]?\d+(?:\.\d+)?)$/) {
$eval .= $1." "; # note: untaints!
}
elsif ($token eq 'plugin') {
@@ -679,43 +679,32 @@
# eval type handling
if (($type & 1) == 1) {
- my @args;
if (my ($function, $args) = ($text =~ m/(.*?)\s*\((.*?)\)\s*$/)) {
- if ($args) {
- # bug 4419: Parse quoted strings, unquoted alphanumerics/floats and
- # both unquoted IPv4 and IPv6 addresses. s// is used so that we can
- # determine whether or not we successfully parsed ALL arguments.
- while ($args =~ s/^\s*(?:['"](.*?)['"]|([\d\.:A-Za-z]+?))\s*(?:,\s*|$)//) {
- if (defined $1) {
- push @args, $1;
- }
- else {
- push @args, $2;
- }
- }
- }
- unshift(@args, $function);
- if ($args) {
- $self->lint_warn("syntax error (unparsable argument: $args) for eval function: $name: $text", $name);
- }
+ my ($packed, $argsref) =
+ $self->pack_eval_method($function, $args, $name, $text);
+
+ if (!$packed) {
+ # we've already warned about this
+ }
elsif ($type == $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS) {
- $conf->{body_evals}->{$priority}->{$name} = \@args;
+ $conf->{body_evals}->{$priority}->{$name} = $packed;
}
elsif ($type == $Mail::SpamAssassin::Conf::TYPE_HEAD_EVALS) {
- $conf->{head_evals}->{$priority}->{$name} = \@args;
+ $conf->{head_evals}->{$priority}->{$name} = $packed;
}
elsif ($type == $Mail::SpamAssassin::Conf::TYPE_RBL_EVALS) {
# We don't do priorities for $Mail::SpamAssassin::Conf::TYPE_RBL_EVALS
- $conf->{rbl_evals}->{$name} = \@args;
+ # we also use the arrayref instead of the packed string
+ $conf->{rbl_evals}->{$name} = [ $function, @$argsref ];
}
elsif ($type == $Mail::SpamAssassin::Conf::TYPE_RAWBODY_EVALS) {
- $conf->{rawbody_evals}->{$priority}->{$name} = \@args;
+ $conf->{rawbody_evals}->{$priority}->{$name} = $packed;
}
elsif ($type == $Mail::SpamAssassin::Conf::TYPE_FULL_EVALS) {
- $conf->{full_evals}->{$priority}->{$name} = \@args;
+ $conf->{full_evals}->{$priority}->{$name} = $packed;
}
#elsif ($type == $Mail::SpamAssassin::Conf::TYPE_URI_EVALS) {
- # $conf->{uri_evals}->{$priority}->{$name} = \@args;
+ # $conf->{uri_evals}->{$priority}->{$name} = $packed;
#}
else {
$self->lint_warn("unknown type $type for $name: $text", $name);
@@ -796,7 +785,7 @@
# Go through each token in the meta rule
foreach my $token (@tokens) {
# has to be an alpha+numeric token
- next if ($token =~ /^(?:\W+|\d+)$/);
+ next if ($token =~ /^(?:\W+|[+-]?\d+(?:\.\d+)?)$/);
# and has to be a rule name
next unless exists $conf->{tests}->{$token};
@@ -836,6 +825,40 @@
}
}
+sub pack_eval_method {
+ my ($self, $function, $args, $name, $text) = @_;
+
+ my @args;
+ if ($args) {
+ # bug 4419: Parse quoted strings, unquoted alphanumerics/floats and
+ # both unquoted IPv4 and IPv6 addresses. s// is used so that we can
+ # determine whether or not we successfully parsed ALL arguments.
+ while ($args =~ s/^\s*(?:['"](.*?)['"]|([\d\.:A-Za-z]+?))\s*(?:,\s*|$)//) {
+ if (defined $1) {
+ push @args, $1;
+ }
+ else {
+ push @args, $2;
+ }
+ }
+ }
+
+ if ($args) {
+ $self->lint_warn("syntax error (unparsable argument: $args) for eval function: $name: $text", $name);
+ return;
+ }
+
+ my $argstr = $function;
+ $argstr =~ s/\s+//gs;
+
+ if (scalar @args > 0) {
+ $argstr .= ',' . join (', ', map {
+ s/\#/[HASH]/gs; "q#".$_."#"
+ } @args);
+ }
+ return ($argstr, \@args);
+}
+
###########################################################################
sub lint_trusted_networks {
@@ -972,7 +995,7 @@
# Go through each token in the meta rule
foreach my $token (@tokens) {
# Numbers can't be rule names
- if ($token =~ /^(?:\W+|\d+)$/) {
+ if ($token =~ /^(?:\W+|[+-]?\d+(?:\.\d+)?)$/) {
$meta .= "$token ";
}
# Zero will probably cause more errors
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Constants.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Constants.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Constants.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Constants.pm Wed Dec 6 06:39:45 2006
@@ -286,7 +286,7 @@
# used for meta rules and "if" conditionals in Conf::Parser
use constant ARITH_EXPRESSION_LEXER => qr/(?:
[\-\+\d\.]+| # A Number
- \w[\w\:]+| # Rule or Class Name
+ \w[\w\:]*| # Rule or Class Name
[\(\)]| # Parens
\|\|| # Boolean OR
\&\&| # Boolean AND
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Message.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Message.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Message.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Message.pm Wed Dec 6 06:39:45 2006
@@ -516,9 +516,36 @@
# Clean ourself up
$self->finish_metadata();
- # wipe out all of our pointers and such -- since our tree is one-way, wiping
- # out body_parts here will wipe out the rest of the tree.
- %{$self} = ();
+ # These will only be in the root Message node
+ delete $self->{'mime_boundary_state'};
+ delete $self->{'mbox_sep'};
+ delete $self->{'normalize'};
+ delete $self->{'pristine_body'};
+ delete $self->{'pristine_headers'};
+ delete $self->{'line_ending'};
+ delete $self->{'missing_head_body_separator'};
+
+ my @toclean = ( $self );
+
+ # Go ahead and clean up all of the Message::Node parts
+ while (my $part = shift @toclean) {
+ delete $part->{'headers'};
+ delete $part->{'raw_headers'};
+ delete $part->{'header_order'};
+ delete $part->{'raw'};
+ delete $part->{'decoded'};
+ delete $part->{'rendered'};
+ delete $part->{'visible_rendered'};
+ delete $part->{'invisible_rendered'};
+ delete $part->{'type'};
+ delete $part->{'rendered_type'};
+
+ # if there are children nodes, add them to the queue of nodes to clean up
+ if (exists $part->{'body_parts'}) {
+ push(@toclean, @{$part->{'body_parts'}});
+ delete $part->{'body_parts'};
+ }
+ }
}
# ---------------------------------------------------------------------------
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/PerMsgStatus.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/PerMsgStatus.pm Wed Dec 6 06:39:45 2006
@@ -222,10 +222,11 @@
eval {
my $learnstatus = $self->{main}->learn ($self->{msg}, undef, $isspam, 0);
- $learnstatus->finish();
if ($learnstatus->did_learn()) {
$self->{auto_learn_status} = $isspam ? "spam" : "ham";
}
+ # This must wait until the did_learn call.
+ $learnstatus->finish();
$self->{main}->finish_learner(); # for now
if (exists $self->{main}->{bayes_scanner}) {
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/Check.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/Check.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/Check.pm Wed Dec 6 06:39:45 2006
@@ -322,7 +322,7 @@
foreach $token (@tokens) {
# Numbers can't be rule names
- if ($token =~ /^(?:\W+|\d+)$/) {
+ if ($token =~ /^(?:\W+|[+-]?\d+(?:\.\d+)?)$/) {
$meta{$rulename} .= "$token ";
}
else {
@@ -552,7 +552,6 @@
{
my ($self, $pms, $conf, $rulename, $pat, %opts) = @_;
my $sub;
-
if (($conf->{tflags}->{$rulename}||'') =~ /\bmultiple\b/)
{
# support multiple matches
@@ -568,7 +567,6 @@
}
}
';
-
}
else {
# omitting the "pos" call, "body_loopid" label, use of while()
@@ -582,7 +580,6 @@
}
}
';
-
}
if ($self->{main}->{use_rule_subs}) {
@@ -609,7 +606,8 @@
$self->add_temporary_method ($rulename.'_body_test',
'{ my $self = shift; '.$sub.' }');
}
- });
+ }
+ );
}
###########################################################################
@@ -819,7 +817,7 @@
sub run_eval_tests {
my ($self, $pms, $testtype, $evalhash, $prepend2desc, $priority, @extraevalargs) = @_;
-
+
return if $self->{main}->call_plugins("have_shortcircuited",
{ permsgstatus => $pms });
@@ -880,7 +878,11 @@
}
}
- my ($function, @args) = @{$test};
+ my ($function, $argstr) = ($test,'');
+ if ($test =~ s/^([^,]+)(,.*)$//gs) {
+ ($function, $argstr) = ($1,$2);
+ }
+
if (!$function) {
warn "rules: error: no function defined for $rulename";
next;
@@ -918,11 +920,6 @@
';
}
- my $argstr = '';
- if (scalar @args > 0) {
- $argstr = ',' . join (', ', map { "q#".$_."#" } @args);
- }
-
$evalstr .= '
eval {
@@ -953,8 +950,7 @@
';
}
- %{$evalhash} = ();
- $self->free_ruleset_source($pms, 'eval', $priority);
+ # don't free the eval ruleset here -- we need it in the compiled code!
# nothing done in the loop, that means no rules
return unless ($evalstr);
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/RelayEval.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/RelayEval.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/RelayEval.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Plugin/RelayEval.pm Wed Dec 6 06:39:45 2006
@@ -94,6 +94,32 @@
return 0;
}
+sub check_for_illegal_ip3 {
+ my ($self, $pms) = @_;
+
+ foreach my $rcvd ( @{$pms->{relays_untrusted}} ) {
+ # (note this might miss some hits if the Received.pm skips any invalid IPs)
+ foreach my $check ( $rcvd->{ip}, $rcvd->{by} ) {
+ return 1 if ($check =~ /^
+ (?:[01257]|(?!127.0.0.)127|22[3-9]|23[0-9]|24[0-9]|25[0-5])\.\d+\.\d+\.\d+
+ $/x);
+ }
+ }
+ return 0;
+}
+sub check_for_illegal_ip2 {
+ my ($self, $pms) = @_;
+
+ foreach my $rcvd ( @{$pms->{relays_untrusted}} ) {
+ # (note this might miss some hits if the Received.pm skips any invalid IPs)
+ foreach my $check ( $rcvd->{ip}, $rcvd->{by} ) {
+ return 1 if ($check =~ /^
+ (?:[01257]|(?!127.0.0.)127|22[3-9]|2[3-9]\d|[12]\d{3,}|[3-9]\d\d+)\.\d+\.\d+\.\d+
+ $/x);
+ }
+ }
+ return 0;
+}
sub check_for_illegal_ip {
my ($self, $pms) = @_;
Modified: spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Util/RegistrarBoundaries.pm
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Util/RegistrarBoundaries.pm?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Util/RegistrarBoundaries.pm (original)
+++ spamassassin/branches/jm_re2c_hacks/lib/Mail/SpamAssassin/Util/RegistrarBoundaries.pm Wed Dec 6 06:39:45 2006
@@ -1,4 +1,5 @@
# The (extremely complex) rules for domain delegation.
+# Note that really, this should be called "RegistryBoundaries"; see bug 4605
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -203,7 +204,7 @@
=item ($hostname, $domain) = split_domain ($fqdn)
Cut a fully-qualified hostname into the hostname part and the domain
-part, splitting at the DNS registrar boundary.
+part, splitting at the DNS registry boundary.
Examples:
Modified: spamassassin/branches/jm_re2c_hacks/masses/extract-results
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/extract-results?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/extract-results (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/extract-results Wed Dec 6 06:39:45 2006
@@ -1,7 +1,7 @@
#!/usr/bin/perl
-# This script extracts the confusion matrix (tp, tn, fp, fn) from the output
-# of the logs-to-c program.
+# This script extracts the confusion matrix (tp, tn, fp, fn) from the
+# output of the fp-fn-statistics program.
#
# This is used by the validate-model script to aggregate the results of a
# cross validation for analysis with the compare-models script.
Modified: spamassassin/branches/jm_re2c_hacks/masses/fp-fn-statistics
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/fp-fn-statistics?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/fp-fn-statistics (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/fp-fn-statistics Wed Dec 6 06:39:45 2006
@@ -1,3 +1,181 @@
-#!/bin/sh
+#!/usr/bin/perl -w
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+
+use Getopt::Long;
+use strict;
+
+use vars qw($opt_cffile $opt_lambda $opt_threshold $opt_scoreset
+ $opt_spam $opt_ham $opt_fplog $opt_fnlog);
+
+$opt_cffile = "../rules";
+$opt_threshold = 5;
+$opt_spam = 'spam.log';
+$opt_ham = 'ham.log';
+$opt_scoreset = 0;
+
+GetOptions("cffile=s", "lambda=f", "threshold=f", "spam=s",
+ "ham=s", "scoreset=i", "fplog=s", "fnlog=s");
+
+# If desired, report false positives and false negatives for analysis
+if (defined $opt_fnlog) { open (FNLOG, ">$opt_fnlog"); }
+if (defined $opt_fplog) { open (FPLOG, ">$opt_fplog"); }
+
+# lambda value for TCR equation, representing the cost of of an FP vs. the
+# cost of a FN. Some example values are: 1 = tagged only, 9 = mailed back
+# to sender asking for token, 999 = blocking or deleting a message.
+#
+# We roughly aim for a value representing "moved to infrequently-read folder".
+
+my $lambda = 50;
+if ($opt_lambda) { $lambda = $opt_lambda; }
+
+use vars qw(%scores %allrules %rules);
+
+readscores();
+
+print "Reading per-message hit stat logs and scores...\n";
+my ($num_spam, $num_ham);
+my ($ga_yy, $ga_ny, $ga_yn, $ga_nn, $yyscore, $ynscore, $nyscore, $nnscore);
+
+readlogs();
+
+evaluate();
+
+# show memory usage before we exit
+# print "Running \"ps aux\"...\n";
+# open(PS, "ps aux|");
+# while(<PS>) {
+# print if $. == 1 || /\b$$\b/;
+# }
+# close(PS);
+
+exit 0;
+
+# arguments are $isspam, $count, \@tests, $msgline;
+sub log_line_count {
+ my $score = 0;
+ $score += $scores{$_} for @{$_[2]};
+
+ if ($_[0]) {
+ $num_spam++;
+ if ($score >= $opt_threshold) {
+ $ga_yy++;
+ $yyscore += $score;
+ }
+ else {
+ $ga_yn++;
+ $ynscore += $score;
+ if (defined $opt_fnlog) {
+ print FNLOG $_[3];
+ }
+ }
+ }
+ else {
+ $num_ham++;
+ if ($score >= $opt_threshold) {
+ #print STDERR "FP: $id\n";
+ $ga_ny++;
+ $nyscore += $score;
+ if (defined $opt_fplog) {
+ print FPLOG $_[3];
+ }
+ }
+ else {
+ $ga_nn++;
+ $nnscore += $score;
+ }
+ }
+}
+
+sub readlogs {
+ my $msgline;
+ my $count = 0;
+ $num_spam = $num_ham = 0;
+
+ $ga_yy = $ga_ny = $ga_yn = $ga_nn = 0;
+ $yyscore = $ynscore = $nyscore = $nnscore = 0.0;
+
+ foreach my $file ($opt_spam, $opt_ham) {
+ open (IN, "<$file") || die "Could not open file '$file': $!";
+
+ my $isspam = ($file eq $opt_spam);
+ my $caught; # 1st parameter of log line
+ my $rules; # 4th parameter of log line
+
+ while (defined($msgline = <IN>)) {
+ ($caught, undef, undef, $rules) = split(' ', $msgline);
+
+ # only take lines starting with Y or .
+ next unless ($caught eq 'Y' || $caught eq '.') && $rules;
+
+ # get tests, but ignore unknown tests and subrules
+ my @tests = grep { defined $scores{$_} && !$allrules{$_}->{issubrule} }
+ split(/,/, $rules);
+
+ # run handler
+ log_line_count($isspam, $count, \@tests, $msgline);
+
+ # increment line
+ $count++;
+ }
+ close IN;
+ }
+}
+
+sub readscores {
+ print "Reading scores from \"$opt_cffile\"...\n";
+ system ("../build/parse-rules-for-masses -d \"$opt_cffile\" -s $opt_scoreset") and die;
+ require "./tmp/rules.pl";
+ %allrules = %rules; # ensure it stays global
+}
+
+sub evaluate {
+ printf ("\n# SUMMARY for threshold %3.1f:\n", $opt_threshold);
+ printf "# Correctly non-spam: %6d %4.2f%%\n",
+ $ga_nn, ($ga_nn / $num_ham) * 100.0;
+ printf "# Correctly spam: %6d %4.2f%%\n",
+ $ga_yy, ($ga_yy / $num_spam) * 100.0;
+ printf "# False positives: %6d %4.2f%%\n",
+ $ga_ny, ($ga_ny / $num_ham) * 100.0;
+ printf "# False negatives: %6d %4.2f%%\n",
+ $ga_yn, ($ga_yn / $num_spam) * 100.0;
+
+ # convert to the TCR metrics used in the published lit
+ my $nspamspam = $ga_yy;
+ my $nspamlegit = $ga_yn;
+ my $nlegitspam = $ga_ny;
+ my $nlegitlegit = $ga_yn;
+ my $nlegit = $num_ham;
+ my $nspam = $num_spam;
+
+ my $werr = ($lambda * $nlegitspam + $nspamlegit)
+ / ($lambda * $nlegit + $nspam);
+
+ my $werr_base = $nspam
+ / ($lambda * $nlegit + $nspam);
+
+ $werr ||= 0.000001; # avoid / by 0
+ my $tcr = $werr_base / $werr;
+
+ my $sr = ($nspamspam / $nspam) * 100.0;
+ my $sp = ($nspamspam / ($nspamspam + $nlegitspam)) * 100.0;
+ printf "# TCR(l=%s): %3.6f SpamRecall: %3.3f%% SpamPrec: %3.3f%%\n",
+ $lambda, $tcr, $sr, $sp;
+}
-exec ./logs-to-c --count $*
Modified: spamassassin/branches/jm_re2c_hacks/masses/logs-to-c
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/logs-to-c?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/logs-to-c (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/logs-to-c Wed Dec 6 06:39:45 2006
@@ -18,58 +18,35 @@
# </...@LICENSE>
use Getopt::Long;
-use vars qw($opt_cffile $opt_count $opt_lambda $opt_threshold
- $opt_spam $opt_ham $opt_fplog $opt_fnlog);
+use strict;
+use vars qw($opt_cffile $opt_spam $opt_ham $opt_scoreset);
$opt_cffile = "../rules";
-$opt_count = 0;
-$opt_threshold = 5;
$opt_spam = 'spam.log';
$opt_ham = 'ham.log';
$opt_scoreset = 0;
-GetOptions("cffile=s", "count", "lambda=f", "threshold=f", "spam=s",
- "ham=s", "scoreset=i", "fplog=s", "fnlog=s");
+GetOptions("cffile=s", "spam=s", "ham=s", "scoreset=i");
-# If desired, report false positives and false negatives for analysis
-if (defined $opt_fnlog) { open (FNLOG, ">$opt_fnlog"); }
-if (defined $opt_fplog) { open (FPLOG, ">$opt_fplog"); }
-
-my $nybias = 10;
-
-# lambda value for TCR equation, representing the cost of of an FP vs. the
-# cost of a FN. Some example values are: 1 = tagged only, 9 = mailed back
-# to sender asking for token, 999 = blocking or deleting a message.
-#
-# We roughly aim for a value representing "moved to infrequently-read folder".
-
-my $lambda = 50;
-if ($opt_lambda) { $lambda = $opt_lambda; }
-
-my $msgline;
my $is_spam = ''; # vec aligned with @tests_hit
my @tests_hit = ();
my %mutable_tests = ();
-use vars qw(%rules %allrules);
+use vars qw(%rules %allrules %scores);
+
+my (%ignored_rule, %range_lo, %range_hi);
+my %rule_to_index;
readscores();
print "Reading per-message hit stat logs and scores...\n";
my ($num_tests, $num_spam, $num_ham);
-my ($ga_yy, $ga_ny, $ga_yn, $ga_nn, $yyscore, $ynscore, $nyscore, $nnscore);
read_ranges();
readlogs();
-if ($opt_count) {
- $nybias = $nybias*($num_spam / $num_ham);
- evaluate();
-}
-else {
- print "Writing logs and current scores as C code...\n";
- writescores_c();
-}
+print "Writing logs and current scores as C code...\n";
+writescores_c();
# show memory usage before we exit
# print "Running \"ps aux\"...\n";
@@ -108,42 +85,6 @@
return map { $short_to_long[$_] } unpack("w*", $_[0]);
}
-# arguments are $isspam, $count, \@tests
-sub log_line_count {
- my $score = 0;
- $score += $scores{$_} for @{$_[2]};
-
- if ($_[0]) {
- $num_spam++;
- if ($score >= $opt_threshold) {
- $ga_yy++;
- $yyscore += $score;
- }
- else {
- $ga_yn++;
- $ynscore += $score;
- if (defined $opt_fnlog) {
- print FNLOG $msgline;
- }
- }
- }
- else {
- $num_ham++;
- if ($score >= $opt_threshold) {
- #print STDERR "FP: $id\n";
- $ga_ny++;
- $nyscore += $score;
- if (defined $opt_fplog) {
- print FPLOG $msgline;
- }
- }
- else {
- $ga_nn++;
- $nnscore += $score;
- }
- }
-}
-
# arguments are $isspam, $count, \@tests;
sub log_line_code {
$tests_hit[$_[1]] = freeze_tests($_[2]);
@@ -159,17 +100,11 @@
}
sub readlogs {
+ my $msgline;
+
my $count = 0;
$num_spam = $num_ham = 0;
- if ($opt_count) {
- $ga_yy = $ga_ny = $ga_yn = $ga_nn = 0;
- $yyscore = $ynscore = $nyscore = $nnscore = 0.0;
- }
-
- # set handler for log lines
- my $log_line = $opt_count ? \&log_line_count : \&log_line_code;
-
foreach my $file ($opt_spam, $opt_ham) {
open (IN, "<$file") || die "Could not open file '$file': $!";
@@ -188,7 +123,7 @@
split(/,/, $rules);
# run handler
- $log_line->($isspam, $count, \@tests);
+ log_line_code($isspam, $count, \@tests);
# increment line
$count++;
@@ -214,11 +149,11 @@
# jm: now, score-ranges-from-freqs has tflags to work from, so
# it will always list all mutable tests.
- @index_to_rule = sort {($ignored_rule{$a} <=> $ignored_rule{$b}) ||
+ my @index_to_rule = sort {($ignored_rule{$a} <=> $ignored_rule{$b}) ||
($mutable_tests{$b} <=> $mutable_tests{$a}) ||
($a cmp $b)} (keys %scores);
my $max_hits_per_msg = 0;
- for ($file = 0; $file < $num_tests; $file++) {
+ for (my $file = 0; $file < $num_tests; $file++) {
my(@hits) =
grep {(! $ignored_rule{$_}) && $mutable_tests{$_}} (thaw_tests($tests_hit[$file]));
if ((scalar(@hits)+1) > $max_hits_per_msg) {
@@ -415,8 +350,6 @@
}
$ignored_rule{$t} = 0;
- $index_to_rule[$count] = $t;
- $count++;
if (!$mut) {
$mutable_tests{$t} = 0;
@@ -428,7 +361,7 @@
$mutable_tests{$t} = 1;
}
unless ($mutable_tests{$t} || $scores{$t}) {
- warn "ignoring '$t': immutable and score == 0\n";
+# warn "ignoring '$t': immutable and score == 0\n";
$ignored_rule{$t} = 1;
}
}
@@ -456,8 +389,6 @@
$ignored_rule{$t} = 1;
}
}
- $index_to_rule[$count] = $t;
- $count++;
}
foreach my $t (keys %range_lo) {
next if ($ignored_rule{$t});
@@ -495,39 +426,6 @@
}
}
-sub evaluate {
- printf ("\n# SUMMARY for threshold %3.1f:\n", $opt_threshold);
- printf "# Correctly non-spam: %6d %4.2f%%\n",
- $ga_nn, ($ga_nn / $num_ham) * 100.0;
- printf "# Correctly spam: %6d %4.2f%%\n",
- $ga_yy, ($ga_yy / $num_spam) * 100.0;
- printf "# False positives: %6d %4.2f%%\n",
- $ga_ny, ($ga_ny / $num_ham) * 100.0;
- printf "# False negatives: %6d %4.2f%%\n",
- $ga_yn, ($ga_yn / $num_spam) * 100.0;
-
- # convert to the TCR metrics used in the published lit
- my $nspamspam = $ga_yy;
- my $nspamlegit = $ga_yn;
- my $nlegitspam = $ga_ny;
- my $nlegitlegit = $ga_yn;
- my $nlegit = $num_ham;
- my $nspam = $num_spam;
-
- my $werr = ($lambda * $nlegitspam + $nspamlegit)
- / ($lambda * $nlegit + $nspam);
-
- my $werr_base = $nspam
- / ($lambda * $nlegit + $nspam);
-
- $werr ||= 0.000001; # avoid / by 0
- my $tcr = $werr_base / $werr;
-
- my $sr = ($nspamspam / $nspam) * 100.0;
- my $sp = ($nspamspam / ($nspamspam + $nlegitspam)) * 100.0;
- printf "# TCR(l=%s): %3.6f SpamRecall: %3.3f%% SpamPrec: %3.3f%%\n",
- $lambda, $tcr, $sr, $sp;
-}
__DATA__
Modified: spamassassin/branches/jm_re2c_hacks/masses/mass-check
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/mass-check?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/mass-check (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/mass-check Wed Dec 6 06:39:45 2006
@@ -149,6 +149,8 @@
$opt_c = "$FindBin::Bin/../rules";
$opt_p = "$FindBin::Bin/spamassassin";
$opt_j = 1;
+$opt_head = 0;
+$opt_tail = 0;
$opt_net = 0;
$opt_hamlog = "ham.log";
$opt_spamlog = "spam.log";
@@ -321,15 +323,13 @@
}
# ArchiveIterator options for non-client mode
- $AIopts->{'opt_n'} = $opt_n;
- $AIopts->{'opt_head'} = $opt_head;
- $AIopts->{'opt_tail'} = $opt_tail;
$AIopts->{'opt_scanprob'} = $opt_scanprob;
$AIopts->{'opt_cache'} = $opt_cache;
$AIopts->{'opt_cachedir'} = $opt_cachedir;
$AIopts->{'opt_after'} = $opt_after;
$AIopts->{'opt_before'} = $opt_before;
$AIopts->{'scan_progress_sub'} = \&showdots_blip;
+ $AIopts->{'opt_want_date'} = ! $opt_n;
# ensure that scanprob stuff is predictable and reproducable
if (defined $opt_scanprob && $opt_scanprob < 1.0) {
@@ -338,7 +338,8 @@
}
else {
# ArchiveIterator options for client mode -- tends to be simple
- $AIopts->{'opt_n'} = 1;
+ $opt_n = 1;
+ $AIopts->{'opt_want_date'} = 0;
}
###########################################################################
@@ -377,12 +378,7 @@
}
elsif (defined $tmpf) {
# child -- process using message_array
- my($num, $messages) = $iter->message_array(\@targets);
-
- # Dump out the number of messages and the message index info to
- # the temp file
- send_line($tmpfd, $num, @{$messages});
-
+ generate_queue(\@targets, $tmpfd);
exit;
}
else {
@@ -391,11 +387,7 @@
}
else {
# we get here if opt_j == 0, so scan in this process
- my($num, $messages) = $iter->message_array(\@targets);
-
- # Dump out the number of messages and the message index info to
- # the temp file
- send_line($tmpfd, $num, @{$messages});
+ generate_queue(\@targets, $tmpfd);
}
# we now have a temporary file with the messages to process
@@ -450,8 +442,6 @@
# message-selection options; these can now be specified separately
# for each target
my %selopts = (
- opt_head => $opt_head,
- opt_tail => $opt_tail,
opt_scanprob => $opt_scanprob,
opt_after => $opt_after,
opt_before => $opt_before
@@ -959,7 +949,7 @@
exit;
}
- my($class, $format, $date, $where, $result) = $iter->run_message($line);
+ my($class, $format, $date, $where, $result) = $iter->_run_message($line);
$result ||= '';
# If determine_receive_date is not set, the original input date
@@ -967,7 +957,7 @@
# the packed version if possible ... use defined for date since
# it could == 0.
if (!$iter->{determine_receive_date} && $class && $format && defined $date && $where) {
- $line = Mail::SpamAssassin::ArchiveIterator::index_pack($date, $class, $format, $where);
+ $line = Mail::SpamAssassin::ArchiveIterator::_index_pack($date, $class, $format, $where);
}
send_line($parent,"$result\0RESULT $line");
@@ -1154,7 +1144,7 @@
send_line($gzfd, $data) || die "mass-check: error when writing to gz temp file\n";
if (!$paths_only) {
- my $msg = ($iter->run_message($data))[4];
+ my $msg = ($iter->_run_message($data))[4];
send_line($gzfd, join('', @{$msg})) ||
die "mass-check: error when writing to gz temp file\n";
}
@@ -1200,7 +1190,7 @@
# process, and we take whatever the first responder sends us.
if (exists $msgsout->{$k}) {
# the result_sub will need parts of the message data, so get it ready
- my @d = Mail::SpamAssassin::ArchiveIterator::index_unpack($msgsout->{$k}->{'data'});
+ my @d = Mail::SpamAssassin::ArchiveIterator::_index_unpack($msgsout->{$k}->{'data'});
# go ahead and do the result
&{$iter->{result_sub}}($d[1], $v, $d[0]);
@@ -1238,7 +1228,7 @@
my $total_count = 0;
while (($total_messages > $total_count) && ($message = read_line($tmpfd))) {
- my($class, undef, $date, undef, $result) = $iter->run_message($message);
+ my($class, undef, $date, undef, $result) = $iter->_run_message($message);
if ($result) {
&{$iter->{result_sub}}($class, $result, $date);
}
@@ -1271,7 +1261,7 @@
}
elsif ($line =~ /^([^\0]*)\0RESULT (.+)$/s) {
my $result = $1;
- my ($date,$class,$type) = Mail::SpamAssassin::ArchiveIterator::index_unpack($2);
+ my ($date,$class,$type) = Mail::SpamAssassin::ArchiveIterator::_index_unpack($2);
aidbg "mass-check: $class, $type, $date\n";
if (defined $opt_restart && ($total_count % $opt_restart) == 0) {
@@ -1739,10 +1729,10 @@
# this is a little tricky -- we need to process the files in the
# path and format we've created, but the original data is needed
# to create a proper result later, so deal with that here.
- my @d = Mail::SpamAssassin::ArchiveIterator::index_unpack($index);
+ my @d = Mail::SpamAssassin::ArchiveIterator::_index_unpack($index);
$real{"$tmpdir/$num"} = \@d;
send_line($tmpfd,
- Mail::SpamAssassin::ArchiveIterator::index_pack($d[0], $d[1], 'f', "$tmpdir/$num")) ||
+ Mail::SpamAssassin::ArchiveIterator::_index_pack($d[0], $d[1], 'f', "$tmpdir/$num")) ||
die "mass-check: error writing out temp file in client mode\n";
}
else {
@@ -1753,7 +1743,7 @@
# in paths_only mode, there's no kluging between formats since we're
# reading the same corpus, however we do still need to track server
# message number to message data so our results will be useable.
- my @d = Mail::SpamAssassin::ArchiveIterator::index_unpack($index);
+ my @d = Mail::SpamAssassin::ArchiveIterator::_index_unpack($index);
$real{$d[3]} = $num;
send_line($tmpfd, $index) ||
die "mass-check: error writing out temp file in client mode\n";
@@ -1868,4 +1858,100 @@
if ($opt_before && $opt_after && $opt_after >= $opt_before) {
die "--before ($opt_before) <= --after ($opt_after) -- conflict!";
}
+}
+
+sub generate_queue {
+ my ($targets, $tmpfd) = @_;
+
+ # scan the targets and get the number and list of messages
+ $iter->_scan_targets($targets,
+ sub {
+ my($self, $date, $class, $format, $mail) = @_;
+ push(@{$self->{$class}}, Mail::SpamAssassin::ArchiveIterator::_index_pack($date, $class, $format, $mail));
+ }
+ );
+
+ # deal with opt_head and opt_tail
+ top_and_tail_messages($iter->{h});
+ top_and_tail_messages($iter->{s});
+
+ my $messages;
+ if ($opt_n) {
+ # OPT_N == 1 means don't bother sorting on message receive date
+
+ # for ease of memory, we'll play with pointers
+ $messages = $iter->{s};
+ undef $iter->{s};
+ push(@{$messages}, @{$iter->{h}});
+ undef $iter->{h};
+ }
+ else {
+ # OPT_N == 0 means sort on message receive date
+
+ # Sort the spam and ham groups by date
+ my @s = @{$iter->{s}};
+ undef $iter->{s};
+ my @h = @{$iter->{h}};
+ undef $iter->{h};
+
+ # interleave ordered spam and ham
+ if (@s && @h) {
+ my $ratio = @s / @h;
+ while (@s && @h) {
+ push @{$messages}, (@s / @h > $ratio) ? (shift @s) : (shift @h);
+ }
+ }
+ # push the rest onto the end
+ push @{$messages}, @s, @h;
+ }
+
+ # head or tail < 0 means crop the total list, negate the value appropriately
+ if ($opt_tail < 0) {
+ splice(@{$messages}, 0, $opt_tail);
+ }
+ if ($opt_head < 0) {
+ splice(@{$messages}, -$opt_head);
+ }
+
+ my $num = $Mail::SpamAssassin::ArchiveIterator::MESSAGES = scalar(@{$messages});
+
+ # Dump out the number of messages and the message index info to
+ # the temp file
+ send_line($tmpfd, $num, @{$messages});
+}
+
+sub top_and_tail_messages {
+ my ($ary) = @_;
+
+ if ($opt_n) {
+ # OPT_N == 1 means don't bother sorting on message receive date
+
+ # head or tail > 0 means crop each list
+ if ($opt_tail > 0) {
+ splice(@{$ary}, 0, -$opt_tail);
+ }
+ if ($opt_head > 0) {
+ splice(@{$ary}, min ($opt_head, scalar @{$ary}));
+ }
+ }
+ else {
+ # OPT_N == 0 means sort on message receive date
+
+ # Sort the spam and ham groups by date
+ my @s = sort { $a cmp $b } @{$ary};
+
+ # head or tail > 0 means crop each list
+ if ($opt_tail > 0) {
+ splice(@s, 0, -$opt_tail);
+ }
+ if ($opt_head > 0) {
+ splice(@s, min ($opt_head, scalar @s));
+ }
+
+ @{$ary} = @s;
+ }
+}
+
+sub min {
+ return ($_[0] < $_[1] ? $_[0] : $_[1]);
}
Modified: spamassassin/branches/jm_re2c_hacks/masses/mk-baseline-results
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/mk-baseline-results?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/mk-baseline-results (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/mk-baseline-results Wed Dec 6 06:39:45 2006
@@ -16,10 +16,10 @@
) > /dev/null 2>&1
gen_fp_fn_report () {
- ./logs-to-c \
+ ./fp-fn-statistics \
--spam=spam-test.log \
--ham=ham-test.log \
- --threshold $1 --count --scoreset=$SCORESET | \
+ --threshold $1 --scoreset=$SCORESET | \
sed -e 's/^Reading.*//' -e '/^$/d'
}
Modified: spamassassin/branches/jm_re2c_hacks/masses/rule-qa/automc/gen_info_xml
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/rule-qa/automc/gen_info_xml?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/rule-qa/automc/gen_info_xml (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/rule-qa/automc/gen_info_xml Wed Dec 6 06:39:45 2006
@@ -42,7 +42,8 @@
my $fastinfo = get_fastinfo($dr, $drdir);
if (!defined $fastinfo) { next; }
- if (!-e "$drdir/fastinfo.xml" || -s "$drdir/fastinfo.xml" == 0) {
+# always rewrite
+ {
open (OUT, ">$drdir/fastinfo.xml") or die "cannot write $drdir/fastinfo.xml";
print OUT $fastinfo;
close OUT or die "failed to write to $drdir/fastinfo.xml";
@@ -138,7 +139,7 @@
open (IN, "<$fname") or warn "cannot read $fname";
while (<IN>) {
# spam results used for 329933 DETAILS all: spam-mc-fast.log spam-foo.log
- if (/^# spam results used for \S+ \S+ \S+: (.*)$/) {
+ if (/^# spam results used for .+?: (.*)$/) {
my $sub = $1;
$sub =~ s/\bspam-/ /gs;
$sub =~ s/\.log\b/ /gs;
Modified: spamassassin/branches/jm_re2c_hacks/masses/runGA
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/masses/runGA?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/masses/runGA (original)
+++ spamassassin/branches/jm_re2c_hacks/masses/runGA Wed Dec 6 06:39:45 2006
@@ -81,9 +81,9 @@
# This needs to have 50_scores.cf in place first ...
echo "[gen test results]"
-./logs-to-c --spam=spam-test.log \
+./fp-fn-statistics --spam=spam-test.log \
--ham=ham-test.log \
- --count --cffile=../rules --scoreset=$SCORESET | tee $LOGDIR/test
+ --cffile=../rules --scoreset=$SCORESET | tee $LOGDIR/test
echo "[STATISTICS file generation]"
./mk-baseline-results $SCORESET | tee $LOGDIR/statistics
Modified: spamassassin/branches/jm_re2c_hacks/rules/20_dnsbl_tests.cf
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/20_dnsbl_tests.cf?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/20_dnsbl_tests.cf (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/20_dnsbl_tests.cf Wed Dec 6 06:39:45 2006
@@ -135,27 +135,25 @@
#reuse RCVD_IN_SORBS_DUL
# ---------------------------------------------------------------------------
-# Spamhaus SBL+XBL
+# Spamhaus SBL+XBL, now called Zen
#
# Spamhaus XBL contains both the Abuseat CBL (cbl.abuseat.org) and Blitzed
# OPM (opm.blitzed.org) lists so it's not necessary to query those as well.
-# TODO: replace with ZEN from rulesrc/sandbox/jm/20_zen.cf. For now,
-# allow both to run for 1 weekend to verify sanity of results...
-header __RCVD_IN_SBL_XBL eval:check_rbl('sblxbl', 'sbl-xbl.spamhaus.org.')
-describe __RCVD_IN_SBL_XBL Received via a relay in Spamhaus SBL+XBL
-tflags __RCVD_IN_SBL_XBL net
+header __RCVD_IN_ZEN eval:check_rbl('zen', 'zen.spamhaus.org.')
+describe __RCVD_IN_ZEN Received via a relay in Spamhaus Zen
+tflags __RCVD_IN_ZEN net
# SBL is the Spamhaus Block List: http://www.spamhaus.org/sbl/
-header RCVD_IN_SBL eval:check_rbl_sub('sblxbl', '127.0.0.2')
-describe RCVD_IN_SBL Received via a relay in Spamhaus SBL
-tflags RCVD_IN_SBL net
+header RCVD_IN_SBL eval:check_rbl_sub('zen', '127.0.0.2')
+describe RCVD_IN_SBL Received via a relay in Spamhaus SBL
+tflags RCVD_IN_SBL net
#reuse RCVD_IN_SBL
# XBL is the Exploits Block List: http://www.spamhaus.org/xbl/
-header RCVD_IN_XBL eval:check_rbl('sblxbl-notfirsthop', 'sbl-xbl.spamhaus.org.', '127.0.0.[456]')
-describe RCVD_IN_XBL Received via a relay in Spamhaus XBL
-tflags RCVD_IN_XBL net
+header RCVD_IN_XBL eval:check_rbl('zen-notfirsthop', 'zen.spamhaus.org.', '127.0.0.[45678]')
+describe RCVD_IN_XBL Received via a relay in Spamhaus XBL
+tflags RCVD_IN_XBL net
#reuse RCVD_IN_XBL
# ---------------------------------------------------------------------------
Modified: spamassassin/branches/jm_re2c_hacks/rules/25_replace.cf
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/25_replace.cf?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/25_replace.cf (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/25_replace.cf Wed Dec 6 06:39:45 2006
@@ -75,7 +75,7 @@
header __SUBJECT_FUZZY_VPILL Subject =~ /<inter W2><post P3>(?!viagra)<V><I><A><G><R><A>/i
replace_rules __SUBJECT_FUZZY_VPILL
meta SUBJECT_FUZZY_VPILL __SUBJECT_FUZZY_VPILL && !FUZZY_VPILL
-describe SUBJECT_FUZZY_MEDS Attempt to obfuscate words in Subject:
+describe SUBJECT_FUZZY_VPILL Attempt to obfuscate words in Subject:
header SUBJECT_FUZZY_CHEAP Subject =~ /<inter W2><post P3>\b(?!cheap)<C><H><E><A><P>(?:\b|<E>)/i
describe SUBJECT_FUZZY_CHEAP Attempt to obfuscate words in Subject:
Modified: spamassassin/branches/jm_re2c_hacks/rules/30_text_de.cf
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/30_text_de.cf?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/30_text_de.cf (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/30_text_de.cf Wed Dec 6 06:39:45 2006
@@ -93,7 +93,6 @@
lang de describe RCVD_IN_SORBS_BLOCK SORBS: Senderechner verweigert Tests
lang de describe RCVD_IN_SORBS_ZOMBIE SORBS: Senderechner in Liste "entführter" Adressblöcke
lang de describe RCVD_IN_SORBS_DUL SORBS: Senderechner nur temporär mit Internet verbunden
-lang de describe __RCVD_IN_SBL_XBL Transportiert via Rechner in SBL+XBL-Liste (http://www.spamhaus.org/)
lang de describe RCVD_IN_SBL Transportiert via Rechner in SBL-Liste (http://www.spamhaus.org/sbl/)
lang de describe RCVD_IN_XBL Transportiert via Rechner in XBL-Liste (http://www.spamhaus.org/xbl/)
lang de describe DNS_FROM_RFC_DSN Absender in dsn-Liste von www.rfc-ignorant.org
Modified: spamassassin/branches/jm_re2c_hacks/rules/30_text_nl.cf
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/30_text_nl.cf?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/30_text_nl.cf (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/30_text_nl.cf Wed Dec 6 06:39:45 2006
@@ -56,7 +56,6 @@
lang nl describe RCVD_IN_SORBS_BLOCK SORBS: verzender weigert getest te worden
lang nl describe RCVD_IN_SORBS_ZOMBIE SORBS: verzender is een gekaapt netwerk
lang nl describe RCVD_IN_SORBS_DUL SORBS: bericht is direct verstuurd vanaf een dynamisch IP adres
-lang nl describe __RCVD_IN_SBL_XBL Ontvangen via een relay die gevonden is in Spamhaus SBL+XBL
lang nl describe RCVD_IN_SBL Ontvangen via een relay die gevonden is in Spamhaus SBL
lang nl describe RCVD_IN_XBL Ontvangen via een relay die gevonden is in Spamhaus XBL
lang nl describe RCVD_IN_DSBL Ontvangen via een relay die gevonden is in list.dsbl.org
Modified: spamassassin/branches/jm_re2c_hacks/rules/50_scores.cf
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/50_scores.cf?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/50_scores.cf (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/50_scores.cf Wed Dec 6 06:39:45 2006
@@ -698,3 +698,14 @@
score MICROSOFT_EXECUTABLE 0.100
score MIME_SUSPECT_NAME 0.100
endif # Mail::SpamAssassin::Plugin::AntiVirus
+
+# VBounce - anti-bounce message ruleset. All of these are informational,
+# and should not be scored as antispam rules; instead the user needs
+# to detect messages that hit ANY_BOUNCE_MESSAGE, and filter them aside
+# as spurious bounces.
+#
+score BOUNCE_MESSAGE 0.1
+score CRBOUNCE_MESSAGE 0.1
+score VBOUNCE_MESSAGE 0.1
+score ANY_BOUNCE_MESSAGE 0.1
+
Modified: spamassassin/branches/jm_re2c_hacks/rules/v320.pre
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/rules/v320.pre?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/rules/v320.pre (original)
+++ spamassassin/branches/jm_re2c_hacks/rules/v320.pre Wed Dec 6 06:39:45 2006
@@ -38,6 +38,10 @@
loadplugin Mail::SpamAssassin::Plugin::URIEval
loadplugin Mail::SpamAssassin::Plugin::WLBLEval
+# VBounce - anti-bounce-message rules, see rules/20_vbounce.cf
+#
+loadplugin Mail::SpamAssassin::Plugin::VBounce
+
# Rule2XSBody - speedup by compilation of ruleset to native code
loadplugin Mail::SpamAssassin::Plugin::Rule2XSBody
Modified: spamassassin/branches/jm_re2c_hacks/sa-learn.raw
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/sa-learn.raw?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/sa-learn.raw (original)
+++ spamassassin/branches/jm_re2c_hacks/sa-learn.raw Wed Dec 6 06:39:45 2006
@@ -380,35 +380,49 @@
close(F);
}
- # add leftover args as targets
- foreach (@ARGV) { target($_); }
+ ###########################################################################
+ # Deal with the target listing, and STDIN -> tempfile
- #No arguments means they want stdin:
- if ( $#targets < 0 ) {
- target('-');
- }
+ my $tempfile; # will be defined if stdin -> tempfile
+ push(@targets, @ARGV);
+ @targets = ('-') unless @targets;
+
+ for(my $elem = 0; $elem <= $#targets; $elem++) {
+ # ArchiveIterator doesn't really like STDIN, so if "-" is specified
+ # as a target, make it a temp file instead.
+ if ( $targets[$elem] =~ /(?:^|:)-$/ ) {
+ if (defined $tempfile) {
+ # uh-oh, stdin specified multiple times?
+ warn "skipping extra stdin target (".$targets[$elem].")\n";
+ splice @targets, $elem, 1;
+ $elem--; # go back to this element again
+ next;
+ }
+ else {
+ my $handle;
- # mbox and mbx doesn't deal with STDIN, so make a temp file if they want STDIN.
- # do it here since they may specify "-" on the commandline
- #
- my $tempfile;
- if ( $targets[0] =~ /:mbo?x:-$/ ) {
- my $handle;
-
- local $/ = undef; # go into slurp mode
- ( $tempfile, $handle ) = Mail::SpamAssassin::Util::secure_tmpfile();
- print {$handle} <STDIN>;
- close $handle;
+ local $/ = undef; # go into slurp mode
+ ( $tempfile, $handle ) = Mail::SpamAssassin::Util::secure_tmpfile();
+ print {$handle} <STDIN>;
+ close $handle;
- # re-aim the targets at the tempfile instead of STDIN
- $targets[0] =~ s/:-$/:$tempfile/;
+ # re-aim the targets at the tempfile instead of STDIN
+ $targets[$elem] =~ s/-$/$tempfile/;
+ }
+ }
+
+ # make sure the target list is in the normal AI format
+ if ($targets[$elem] !~ /^[^:]*:[a-z]+:/) {
+ $targets[$elem] = target($targets[$elem]);
+ }
}
+ ###########################################################################
+
my $iter = new Mail::SpamAssassin::ArchiveIterator(
{
- 'opt_j' => 0,
- 'opt_n' => 1,
'opt_all' => 0, # skip messages over 250k
+ 'opt_want_date' => 0,
}
);
Modified: spamassassin/branches/jm_re2c_hacks/sa-update.raw
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/sa-update.raw?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/sa-update.raw (original)
+++ spamassassin/branches/jm_re2c_hacks/sa-update.raw Wed Dec 6 06:39:45 2006
@@ -839,6 +839,7 @@
die "fatal: can't access $UPDDir: $!\n";
}
while(my $file = readdir(DIR)) {
+ local ($1); # avoid random taint flagging of $1
$file =~ /^(.+)$/; # untaint
$file = $1;
my $path = File::Spec->catfile($UPDDir, $file);
@@ -1238,6 +1239,7 @@
return;
}
while(my $file = readdir(DIR)) {
+ local ($1); # bug 5216: prevent random taint flagging of $1
$file =~ /^(.+)$/; # untaint
$file = $1;
Modified: spamassassin/branches/jm_re2c_hacks/spamassassin.raw
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/spamassassin.raw?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/spamassassin.raw (original)
+++ spamassassin/branches/jm_re2c_hacks/spamassassin.raw Wed Dec 6 06:39:45 2006
@@ -82,8 +82,10 @@
use Mail::SpamAssassin::Logger qw(log_message);
BEGIN {
- # redirect __WARN__ and __DIE__, but NOT until after the
+ # redirect __WARN__, but NOT until after the
# Mail::SpamAssassin::Logger class has been parsed.
+ # do not trap warnings here based on eval scope; evals are very
+ # common throughout. die()s can be trapped in future though.
$SIG{__WARN__} = sub {
log_message("warn", $_[0]);
};
@@ -308,40 +310,54 @@
}
###########################################################################
+# Deal with the target listing, and STDIN -> tempfile
+
+my $tempfile; # will be defined if stdin -> tempfile
+push(@targets, @ARGV);
+@targets = ('-') unless @targets;
+
+for(my $elem = 0; $elem <= $#targets; $elem++) {
+ # ArchiveIterator doesn't really like STDIN, so if "-" is specified
+ # as a target, make it a temp file instead.
+ if ( $targets[$elem] =~ /(?:^|:)-$/ ) {
+ if (defined $tempfile) {
+ # uh-oh, stdin specified multiple times?
+ warn "skipping extra stdin target (".$targets[$elem].")\n";
+ splice @targets, $elem, 1;
+ $elem--; # go back to this element again
+ next;
+ }
+ else {
+ my $handle;
+
+ local $/ = undef; # go into slurp mode
+ ( $tempfile, $handle ) = Mail::SpamAssassin::Util::secure_tmpfile();
+ print {$handle} <STDIN>;
+ close $handle;
+
+ # re-aim the targets at the tempfile instead of STDIN
+ $targets[$elem] =~ s/-$/$tempfile/;
+ }
+ }
+
+ # make sure the target list is in the normal AI format
+ if ($targets[$elem] !~ /^[^:]*:[a-z]+:/) {
+ my $format = $opt{'format'} || 'detect';
+ $targets[$elem] = join ( ":", '', $format, $targets[$elem] );
+ }
+}
+
+###########################################################################
# Everything below here needs ArchiveIterator ...
my $iter = new Mail::SpamAssassin::ArchiveIterator(
{
- 'opt_j' => 0,
- 'opt_n' => 1,
'opt_all' => 1,
'opt_want_date' => 0
}
);
$iter->set_functions( \&wanted, \&result );
-
-# add leftover args as targets
-# no arguments means they want stdin:
-push ( @ARGV, '-' ) if ( !@ARGV );
-@targets = map { join ( ":", '', $opt{'format'}, $_ ) } @ARGV;
-
-# mbox and mbx handling doesn't deal with STDIN, so make a temp file if they
-# want STDIN. do it here since they may specify "-" on the commandline
-# instead of leaving it as the default.
-#
-my $tempfile;
-if ( $targets[0] =~ /:mbo?x:-$/ ) {
- my $handle;
-
- local $/ = undef; # go into slurp mode
- ( $tempfile, $handle ) = Mail::SpamAssassin::Util::secure_tmpfile();
- print {$handle} <STDIN>;
- close $handle;
-
- # re-aim the targets at the tempfile instead of STDIN
- $targets[0] =~ s/:-$/:$tempfile/;
-}
# Go run the messages!
# bug 4930: use a temp variable since "||=" decides whether or not to set the
Modified: spamassassin/branches/jm_re2c_hacks/spamd/spamd.raw
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/spamd/spamd.raw?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/spamd/spamd.raw (original)
+++ spamassassin/branches/jm_re2c_hacks/spamd/spamd.raw Wed Dec 6 06:39:45 2006
@@ -111,11 +111,14 @@
eval { require Time::HiRes };
Time::HiRes->import(qw(time)) unless $@;
# redirect __WARN__ and __DIE__
+ # do not trap warnings here based on eval scope; evals are very
+ # common throughout. die()s can be trapped though.
$SIG{__WARN__} = sub {
log_message("warn", $_[0]);
};
$SIG{__DIE__} = sub {
- log_message("error", $_[0]) if $_[0] !~ /\bin eval\b/;
+ # see http://use.perl.org/comments.pl?mode=flat&sid=33872 for $^S
+ log_message("error", $_[0]) unless $^S;
};
}
Modified: spamassassin/branches/jm_re2c_hacks/tools/split_corpora
URL: http://svn.apache.org/viewvc/spamassassin/branches/jm_re2c_hacks/tools/split_corpora?view=diff&rev=483095&r1=483094&r2=483095
==============================================================================
--- spamassassin/branches/jm_re2c_hacks/tools/split_corpora (original)
+++ spamassassin/branches/jm_re2c_hacks/tools/split_corpora Wed Dec 6 06:39:45 2006
@@ -55,8 +55,6 @@
my $current_bucket = 0;
my $iter = new Mail::SpamAssassin::ArchiveIterator({
- 'opt_j' => 1,
- 'opt_n' => 1,
'opt_all' => 1,
});