You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2009/06/12 00:49:34 UTC
svn commit: r783951 - /spamassassin/trunk/masses/rule-qa/reports-from-logs
Author: jm
Date: Thu Jun 11 22:49:33 2009
New Revision: 783951
URL: http://svn.apache.org/viewvc?rev=783951&view=rev
Log:
refactor and tidy reports-from-logs; clean up the code quite a lot, and hopefully this will fix that double-counting bug guenther spotted
Modified:
spamassassin/trunk/masses/rule-qa/reports-from-logs
Modified: spamassassin/trunk/masses/rule-qa/reports-from-logs
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/rule-qa/reports-from-logs?rev=783951&r1=783950&r2=783951&view=diff
==============================================================================
--- spamassassin/trunk/masses/rule-qa/reports-from-logs (original)
+++ spamassassin/trunk/masses/rule-qa/reports-from-logs Thu Jun 11 22:49:33 2009
@@ -62,6 +62,8 @@
clean_up();
exit;
+# ---------------------------------------------------------------------------
+
sub configure {
# does rough equivalent of source
open(C, $configuration) || die "open failed: $configuration: $!\n";
@@ -70,23 +72,30 @@
# add 'override' options
my @lines = (<C>, split(/\|/, $opt_override));
- foreach $_ (@lines) {
- chomp;
- s/#.*//;
- if (/^\s*(.*?)\s*=\s*(.*?)\s*$/) {
- my ($key, $val) = ($1, $2);
- $val =~ s/\$PWD/$pwd/gs;
- $cf{$key} = $val;
- }
+ foreach (@lines) {
+ chomp;
+ s/#.*//;
+ if (/^\s*(.*?)\s*=\s*(.*?)\s*$/) {
+ my ($key, $val) = ($1, $2);
+ $val =~ s/\$PWD/$pwd/gs;
+ $cf{$key} = $val;
+ }
}
close(C);
+
+ $cf{output_classes} ||=
+ "DETAILS.new DETAILS.all DETAILS.age NET.new NET.all NET.age";
}
+# ---------------------------------------------------------------------------
+
sub clean_up {
chdir "/";
system "rm -rf $cf{tmp}/*.$$ ".join(' ', @tmps);
}
+# ---------------------------------------------------------------------------
+
sub init {
$SIG{INT} = \&clean_up;
$SIG{TERM} = \&clean_up;
@@ -96,6 +105,8 @@
$ENV{TZ} = 'UTC';
}
+# ---------------------------------------------------------------------------
+
sub locate_input {
opendir(CORPUS, $logsdir);
my @files = sort readdir(CORPUS);
@@ -162,6 +173,8 @@
}
}
+# ---------------------------------------------------------------------------
+
sub sort_all {
my ($a1, $a2) = ($a =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
my ($b1, $b2) = ($b =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
@@ -178,13 +191,17 @@
return $n;
}
+# ---------------------------------------------------------------------------
+
sub time_filter_fileset {
my ($fileary, $outname, $after, $before) = @_;
- my $timet_before = (defined $before ? ($time_start - ($before * WEEK)) : $time_start+1);
- my $timet_after = (defined $after ? ($time_start - ($after * WEEK)) : 0);
+ my $timet_before = (defined $before ?
+ ($time_start - ($before * WEEK)) : $time_start+1);
+ my $timet_after = (defined $after ?
+ ($time_start - ($after * WEEK)) : 0);
- open(TMP, "> $outname") or warn "cannot write $outname";
+ open(TMP, ">$outname") or warn "cannot write $outname";
for my $file (@{$fileary}) {
open(IN, $file) or warn "cannot read $file";
while (<IN>) {
@@ -197,12 +214,10 @@
close TMP or warn "failed to close $outname";
}
+# ---------------------------------------------------------------------------
sub generate_logs {
- my $classes = $cf{output_classes};
- $classes ||= "DETAILS.new DETAILS.all DETAILS.age NET.new NET.all NET.age";
-
- foreach my $entry (split(' ', $classes)) {
+ foreach my $entry (split(' ', $cf{output_classes})) {
$entry =~ /^(\S+)\.(\S+)$/;
my $class = $1;
my $rtype = $2;
@@ -223,29 +238,30 @@
gen_class ($daterev, $rev, $class, $rtype);
}
}
-}
+}
+
+# ---------------------------------------------------------------------------
+
+my ($tmp_h, $tmp_s, $no_messages_in_freqs, $hf_flags);
sub gen_class {
my ($daterev, $rev, $class, $rtype) = @_;
+ return if ($class eq "NET" && $rtype !~ /^(?:new|all|age|7day)$/);
+ chdir $logsdir;
print STDERR "\ngenerating: $cf{html}/$daterev/$class.$rtype\n";
- return if ($class eq "NET" && $rtype !~ /^(?:new|all|age|7day)$/);
-
my @ham = grep { /^ham/ } @{$logs_by_daterev{$daterev}};
- my @spam = grep { /^spam/ } @{$logs_by_daterev{$daterev}};
-
print STDERR "input h: " . join(' ', @ham) . "\n";
- print STDERR "input s: " . join(' ', @spam) . "\n";
- chdir $logsdir;
+ my @spam = grep { /^spam/ } @{$logs_by_daterev{$daterev}};
+ print STDERR "input s: " . join(' ', @spam) . "\n";
# net vs. local
if ($class eq "NET") {
@ham = grep { /-net-/ } @ham;
@spam = grep { /-net-/ } @spam;
}
-
# age
if ($rtype =~ /(\d+)day/) {
my $mtime = $1;
@@ -253,7 +269,6 @@
@spam = grep { -M $_ < $mtime } @spam;
}
-
print STDERR "selected h: " . join(' ', @ham) . "\n";
print STDERR "selected s: " . join(' ', @spam) . "\n";
@@ -295,19 +310,7 @@
my $bytes = 0;
if ($class eq 'LOGS') {
- foreach my $f (@ham, @spam) {
- $f =~ s/[^-\._A-Za-z0-9]+/_/gs; # sanitize!
- my $zf = "$fname-$f.gz";
-
- system("gzip -c < $f > $zf.$$");
- if ($? >> 8 != 0) {
- warn "gzip -c < $f > $zf.$$ failed";
- }
-
- rename("$zf.$$", $zf) or
- warn "cannot rename $zf.$$ to $zf";
- $bytes += (-s $zf);
- }
+ $bytes = gen_report_logs($fname, \@ham, \@spam);
}
else {
my $tmpfname = "$fname.$$";
@@ -319,130 +322,41 @@
push (@tmps, abs_path($tmpfname));
- my $flags = "";
- $flags = "-t net -s 1" if $class eq "NET";
- $flags = "-o" if $class eq "OVERLAP";
- $flags = "-S" if $class eq "SCOREMAP";
+ $hf_flags = "";
+ $hf_flags = "-t net -s 1" if $class eq "NET";
+ $hf_flags = "-o" if $class eq "OVERLAP";
+ $hf_flags = "-S" if $class eq "SCOREMAP";
+
if ($cf{rules_dir}) {
- $flags .= " -c '$cf{rules_dir}'";
+ $hf_flags .= " -c '$cf{rules_dir}'";
}
# are we analyzing --net mass-check logs? if so, use scoreset 1
if (join(" ", @ham) =~ /-net-/) {
- $flags .= " -s 1" if $class eq "NET";
+ $hf_flags .= " -s 1" if $class eq "NET";
}
# catch an odd error condition, where hit-frequencies creates output
# with no log lines included at all
- my $no_msgs = 0;
-
- if ($rtype eq "all") {
- my %spam;
- my %ham;
- my @output;
-
- for my $file (@spam) {
- # ham-someuser.log, ham-net-bb-jm.20090518-r775863-n.log
- my $u = $file;
- $u =~ s/\.log$//; $u =~ s/.*\///; $u =~ s/^(h|sp)am-(?:net-)?//;
- $u =~ s/\.\d{8}-r\d+-[a-z]//; # daterev
- $spam{$u} = $file;
- print "username in spam log: $u\n";
- }
- for my $file (@ham) {
- my $u = $file;
- $u =~ s/\.log$//; $u =~ s/.*\///; $u =~ s/^(h|sp)am-(?:net-)?//;
- $u =~ s/\.\d{8}-r\d+-[a-z]//; # daterev
- $ham{$u} = $file;
- print "username in ham log: $u\n";
- }
-
- if (scalar keys %spam <= 0 && scalar keys %ham <= 0) {
- warn "no files found for $class.$rtype";
- return;
- }
+ $no_messages_in_freqs = 0;
- unlink "$cf{tmp}/ham.log.$$", "$cf{tmp}/spam.log.$$";
- for my $user (sort keys %spam) {
- my $hamfiles = [];
- if ($ham{$user}) { $hamfiles = [ "$logsdir/$ham{$user}" ]; }
-
- time_filter_fileset($hamfiles,
- "$cf{tmp}/ham.log.$$", $OLDEST_HAM_WEEKS, undef);
- time_filter_fileset([ "$logsdir/$spam{$user}" ],
- "$cf{tmp}/spam.log.$$", $OLDEST_SPAM_WEEKS, undef);
-
- start_freqs($rev, "$flags $cf{tmp}/spam.log.$$ $cf{tmp}/ham.log.$$");
-
- while(<FREQS>) {
- chomp;
- push @output, "$_:$user\n";
- }
- close(FREQS);
-
- system("cat $cf{tmp}/ham.log.$$ >> $cf{tmp}/hamall.log.$$");
- system("cat $cf{tmp}/spam.log.$$ >> $cf{tmp}/spamall.log.$$");
- }
-
- if (-z "$cf{tmp}/hamall.log.$$" && -z "$cf{tmp}/spamall.log.$$") {
- warn "time_filter_fileset() returned empty logs. not creating freqs!";
- return; # we'll try again later
- }
-
- start_freqs($rev, "$flags $cf{tmp}/spamall.log.$$ $cf{tmp}/hamall.log.$$");
-
- while(<FREQS>) {
- /\s0\s+0\s+0.500\s+0.00\s+0.00\s+\(all messages\)/ and $no_msgs = 1;
- push @output, $_;
- }
- close(FREQS);
+ $tmp_h = "$cf{tmp}/ham.log.$$";
+ $tmp_s = "$cf{tmp}/spam.log.$$";
- for (sort sort_all @output) { print OUT; }
+ if ($rtype eq "all") {
+ gen_report_freqs_all($tmpfname, \@ham, \@spam, $rev);
}
elsif ($rtype eq "age") {
- my @output;
-
- for my $which (("0-1", "1-2", "2-3", "3-6")) {
- my ($before, $after) = split(/-/, $which);
- # get and filter logs
- chdir $logsdir;
- time_filter_fileset(\@ham, "$cf{tmp}/ham.log.$$", $after, $before);
- time_filter_fileset(\@spam, "$cf{tmp}/spam.log.$$", $after, $before);
-
- # print out by age
- start_freqs($rev, "$flags $cf{tmp}/spam.log.$$ $cf{tmp}/ham.log.$$");
-
- while(<FREQS>) {
- chomp;
- push @output, "$_:$which\n";
- }
- close(FREQS);
- }
- for (sort sort_all @output) { print OUT; }
+ gen_report_freqs_age($tmpfname, \@ham, \@spam, $rev);
}
elsif (@ham && @spam) {
- # get logs
- time_filter_fileset(\@ham, "$cf{tmp}/ham.log.$$", $OLDEST_HAM_WEEKS, undef);
- time_filter_fileset(\@spam, "$cf{tmp}/spam.log.$$", $OLDEST_SPAM_WEEKS, undef);
-
- if (-z "$cf{tmp}/ham.log.$$" && -z "$cf{tmp}/spam.log.$$") {
- warn "time_filter_fileset() returned empty logs. not creating freqs!";
- return; # we'll try again later
- }
-
- start_freqs($rev, "$flags $cf{tmp}/spam.log.$$ $cf{tmp}/ham.log.$$");
-
- while(<FREQS>) {
- /\s0\s+0\s+0.500\s+0.00\s+0.00\s+\(all messages\)/ and $no_msgs = 1;
- print(OUT);
- }
- close(FREQS);
+ gen_report_freqs_basic($tmpfname, \@ham, \@spam, $rev);
}
$bytes = (-s OUT);
close(OUT);
- if ($no_msgs) {
+ if ($no_messages_in_freqs) {
warn "ERROR: no data in freqs! aborting, leaving tmp file as $tmpfname";
return;
}
@@ -468,11 +382,15 @@
}
+# ---------------------------------------------------------------------------
+
sub mk_daterev {
my ($timet, $rev, $tag) = @_;
return strftime("%Y%m%d", gmtime($timet + DATEREV_ADJ)) . "/r$rev-$tag";
}
+# ---------------------------------------------------------------------------
+
sub create_outputdir {
my ($revpath) = @_;
my $dir = $cf{html} .'/'. $revpath;
@@ -490,6 +408,8 @@
return $dir;
}
+# ---------------------------------------------------------------------------
+
sub log_metadata_xml {
my ($daterev, @files) = @_;
my $str = '';
@@ -511,6 +431,8 @@
return '<mclogmds>'.$str.'</mclogmds>';
}
+# ---------------------------------------------------------------------------
+
sub create_rulemetadata_dir {
my $rev = shift;
my $dir = "$cf{html}/rulemetadata/$rev";
@@ -522,6 +444,8 @@
return $dir;
}
+# ---------------------------------------------------------------------------
+
sub get_rulemetadata_for_revision {
my ($daterev, $rev) = @_;
@@ -539,9 +463,13 @@
if ($? >> 8 != 0) {
warn "'$cmd' failed";
}
+
+ chdir $logsdir;
}
-sub start_freqs {
+# ---------------------------------------------------------------------------
+
+sub start_hit_frequencies_at_rev {
my ($rev, $args) = @_;
$rev ||= 'HEAD';
@@ -567,5 +495,144 @@
chdir "$hfdir/masses" or die "cannot chdir $hfdir/masses";
open (FREQS, "./hit-frequencies -TxpaP $args |")
or die "cannot run ./hit-frequencies $args |";
+
+ chdir $logsdir;
+}
+
+# ---------------------------------------------------------------------------
+
+sub gen_report_logs {
+ my ($fname, $hamref, $spamref) = @_;
+
+ my $bytes = 0;
+ foreach my $f (@$hamref, @$spamref) {
+ $f =~ s/[^-\._A-Za-z0-9]+/_/gs; # sanitize!
+ my $zf = "$fname-$f.gz";
+
+ system("gzip -c < $f > $zf.$$");
+ if ($? >> 8 != 0) {
+ warn "gzip -c < $f > $zf.$$ failed";
+ }
+
+ rename("$zf.$$", $zf) or
+ warn "cannot rename $zf.$$ to $zf";
+ $bytes += (-s $zf);
+ }
+ return $bytes;
+}
+
+# ---------------------------------------------------------------------------
+
+sub gen_report_freqs_all {
+ my ($tmpfname, $hamref, $spamref, $rev) = @_;
+ my %spam;
+ my %ham;
+ my @output;
+
+ for my $file (@$spamref) {
+ my $u = extract_username_from_log_filename($file);
+ $spam{$u} = $file; print "username in spam log: $u\n";
+ }
+ for my $file (@$hamref) {
+ my $u = extract_username_from_log_filename($file);
+ $ham{$u} = $file; print "username in ham log: $u\n";
+ }
+
+ if (scalar keys %spam <= 0 && scalar keys %ham <= 0) {
+ warn "no files found";
+ return;
+ }
+
+ my $tmp_h_all = "$cf{tmp}/hamall.log.$$";
+ my $tmp_s_all = "$cf{tmp}/spamall.log.$$";
+ unlink $tmp_h_all, $tmp_s_all;
+
+ my %alluserkeys;
+ for my $k (keys %spam, keys %ham) {
+ next if exists $alluserkeys{$k}; undef $alluserkeys{$k};
+ }
+ for my $user (sort keys %alluserkeys) {
+ my $files_h = [];
+ if ($ham{$user}) { $files_h = [ "$logsdir/$ham{$user}" ]; }
+ my $files_s = [];
+ if ($spam{$user}) { $files_s = [ "$logsdir/$spam{$user}" ]; }
+
+ time_filter_fileset($files_h, $tmp_h, $OLDEST_HAM_WEEKS, undef);
+ time_filter_fileset($files_s, $tmp_s, $OLDEST_SPAM_WEEKS, undef);
+
+ start_hit_frequencies_at_rev($rev, "$hf_flags $tmp_s $tmp_h");
+ while(<FREQS>) {
+ chomp; push @output, "$_:$user\n";
+ }
+ close(FREQS);
+
+ system("cat $tmp_h >> $tmp_h_all");
+ system("cat $tmp_s >> $tmp_s_all");
+ }
+
+ if (-z $tmp_h_all && -z $tmp_s_all) {
+ warn "time_filter_fileset() returned empty logs. not creating freqs!";
+ return; # we'll try again later
+ }
+
+ start_hit_frequencies_at_rev($rev, "$hf_flags $tmp_s_all $tmp_h_all");
+ while(<FREQS>) {
+ /\s0\s+0\s+0.500\s+0.00\s+0.00\s+\(all messages\)/ and $no_messages_in_freqs = 1;
+ push @output, $_;
+ }
+ close(FREQS);
+ for (sort sort_all @output) { print OUT; }
+}
+
+# ---------------------------------------------------------------------------
+
+sub extract_username_from_log_filename {
+ my $u = shift;
+ # spam-someuser.log, spam-net-bb-jm.20090518-r775863-n.log
+ $u =~ s/\.log$//; $u =~ s/.*\///; $u =~ s/^(h|sp)am-(?:net-)?//;
+ $u =~ s/\.\d{8}-r\d+-[a-z]//; # daterev
+ return $u;
+}
+
+# ---------------------------------------------------------------------------
+
+sub gen_report_freqs_age {
+ my ($tmpfname, $hamref, $spamref, $rev) = @_;
+ my @output;
+
+ for my $which (("0-1", "1-2", "2-3", "3-6")) {
+ my ($before, $after) = split(/-/, $which);
+ time_filter_fileset($hamref, $tmp_h, $after, $before);
+ time_filter_fileset($spamref, $tmp_s, $after, $before);
+
+ # print out by age
+ start_hit_frequencies_at_rev($rev, "$hf_flags $tmp_s $tmp_h");
+ while(<FREQS>) {
+ chomp; push @output, "$_:$which\n";
+ }
+ close(FREQS);
+ }
+ for (sort sort_all @output) { print OUT; }
+}
+
+# ---------------------------------------------------------------------------
+
+sub gen_report_freqs_basic {
+ my ($tmpfname, $hamref, $spamref, $rev) = @_;
+
+ time_filter_fileset($hamref, $tmp_h, $OLDEST_HAM_WEEKS, undef);
+ time_filter_fileset($spamref, $tmp_s, $OLDEST_SPAM_WEEKS, undef);
+
+ if (-z $tmp_h && -z $tmp_s) {
+ warn "time_filter_fileset() returned empty logs. not creating freqs!";
+ return; # we'll try again later
+ }
+
+ start_hit_frequencies_at_rev($rev, "$hf_flags $tmp_s $tmp_h");
+ while(<FREQS>) {
+ /\s0\s+0\s+0.500\s+0.00\s+0.00\s+\(all messages\)/ and $no_messages_in_freqs = 1;
+ print(OUT);
+ }
+ close(FREQS);
}