You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2005/01/27 04:15:02 UTC
svn commit: r126574 - /spamassassin/trunk/masses/rule-qa/corpus-hourly
Author: jm
Date: Wed Jan 26 19:15:02 2005
New Revision: 126574
URL: http://svn.apache.org/viewcvs?view=rev&rev=126574
Log:
refactor out a subroutine from corpus-hourly, so it can support user-specified sets of output files
Modified:
spamassassin/trunk/masses/rule-qa/corpus-hourly
Modified: spamassassin/trunk/masses/rule-qa/corpus-hourly
Url: http://svn.apache.org/viewcvs/spamassassin/trunk/masses/rule-qa/corpus-hourly?view=diff&rev=126574&p1=spamassassin/trunk/masses/rule-qa/corpus-hourly&r1=126573&p2=spamassassin/trunk/masses/rule-qa/corpus-hourly&r2=126574
==============================================================================
--- spamassassin/trunk/masses/rule-qa/corpus-hourly (original)
+++ spamassassin/trunk/masses/rule-qa/corpus-hourly Wed Jan 26 19:15:02 2005
@@ -158,193 +158,204 @@
}
sub current {
- for my $class ("DETAILS", "HTML", "NET") {
-# for my $age ("new", "all", "age", "1day", "2day", "7day") {
- for my $age ("new", "all", "age") {
- print STDERR "generating $class.$age\n";
-
- next if ($class eq "NET" && $age !~ /^(?:new|all|age|7day)$/);
-
- my @ham = grep { /^ham/ } @files;
- my @spam = grep { /^spam/ } @files;
-
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
-
- chdir $opt{corpus};
-
- # net vs. local
- if ($class eq "NET") {
- @ham = grep { /-net-/ } @ham;
- @spam = grep { /-net-/ } @spam;
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
- }
- else {
- # if both net and local exist, use newer
- my %spam;
- my %ham;
-
- for my $file (@spam) {
- $spam{$1}++ if ($file =~ m/-(\w+)\.log$/);
- }
- for my $file (@ham) {
- $ham{$1}++ if ($file =~ m/-(\w+)\.log$/);
- }
- while (my ($user, $count) = each %ham) {
- if ($count > 1) {
- my $nightly = "ham-$user.log";
- my $weekly = "ham-net-$user.log";
- if ($revision{$nightly} >= $revision{$weekly}) {
- @ham = grep { $_ ne $weekly } @ham;
- }
- else {
- @ham = grep { $_ ne $nightly } @ham;
- }
- }
- }
- while (my ($user, $count) = each %spam) {
- if ($count > 1) {
- my $nightly = "spam-$user.log";
- my $weekly = "spam-net-$user.log";
- if ($revision{$nightly} >= $revision{$weekly}) {
- @spam = grep { $_ ne $weekly } @spam;
- }
- else {
- @spam = grep { $_ ne $nightly } @spam;
- }
- }
- }
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
- }
-
- # age
- if ($class eq "NET" && $age ne "7day") {
- @ham = grep { -M "$_" < 10 } @ham;
- @spam = grep { -M "$_" < 10 } @spam;
- # find most recent CVS revision
- my $wanted = 0.0;
- for (@spam, @ham) {
- $wanted = $revision{$_} if ($revision{$_} > $wanted);
- }
- @spam = grep { $revision{$_} eq $wanted } @spam;
- @ham = grep { $revision{$_} eq $wanted } @ham;
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
- }
- elsif ($age =~ /^(?:new|all|age)$/) {
- @ham = grep { -M "$_" < -M $opt{tagtime} } @ham;
- @spam = grep { -M "$_" < -M $opt{tagtime} } @spam;
- if (!$opt{ignore_revisions}) {
- @ham = grep { $revision{$_} eq $revision } @ham;
- @spam = grep { $revision{$_} eq $revision } @spam;
+ my $classes = $opt{output_classes};
+ $classes ||= "DETAILS.new DETAILS.all DETAILS.age HTML.new HTML.all HTML.age NET.new NET.all NET.age";
+
+ foreach my $entry (split(' ', $classes)) {
+ $entry =~ /^(\S+)\.(\S+)$/;
+ my $class = $1;
+ my $age = $2;
+ if (!$age) { warn "no age in $entry"; next; }
+ gen_class ($class, $age);
+ }
+}
+
+sub gen_class {
+ my ($class, $age) = @_;
+
+ print STDERR "generating $class.$age\n";
+
+ next if ($class eq "NET" && $age !~ /^(?:new|all|age|7day)$/);
+
+ my @ham = grep { /^ham/ } @files;
+ my @spam = grep { /^spam/ } @files;
+
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+
+ chdir $opt{corpus};
+
+ # net vs. local
+ if ($class eq "NET") {
+ @ham = grep { /-net-/ } @ham;
+ @spam = grep { /-net-/ } @spam;
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+ }
+ else {
+ # if both net and local exist, use newer
+ my %spam;
+ my %ham;
+
+ for my $file (@spam) {
+ $spam{$1}++ if ($file =~ m/-(\w+)\.log$/);
+ }
+ for my $file (@ham) {
+ $ham{$1}++ if ($file =~ m/-(\w+)\.log$/);
+ }
+ while (my ($user, $count) = each %ham) {
+ if ($count > 1) {
+ my $nightly = "ham-$user.log";
+ my $weekly = "ham-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @ham = grep { $_ ne $weekly } @ham;
+ }
+ else {
+ @ham = grep { $_ ne $nightly } @ham;
+ }
+ }
+ }
+ while (my ($user, $count) = each %spam) {
+ if ($count > 1) {
+ my $nightly = "spam-$user.log";
+ my $weekly = "spam-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @spam = grep { $_ ne $weekly } @spam;
}
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
- }
- elsif ($age =~ /(\d+)day/) {
- my $mtime = $1;
- @ham = grep { -M "$_" < $mtime } @ham;
- @spam = grep { -M "$_" < $mtime } @spam;
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
- }
-
- open(OUT, "> $opt{html}/$class.$age");
- print OUT "# ham results used: " . join(" ", @ham) . "\n";
- print OUT "# spam results used: " . join(" ", @spam) . "\n";
- for (@ham) {
- print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
- }
- for (@spam) {
- print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
- }
-
- my $flags = "";
- $flags = "-t net -s 1" if $class eq "NET";
- $flags = "-M HTML_MESSAGE" if $class eq "HTML";
-
- if ($age eq "all") {
- my %spam;
- my %ham;
- my @output;
-
- for my $file (@spam) {
- $spam{$1} = $file if ($file =~ m/-(\w+)\.log$/);
- }
- for my $file (@ham) {
- $ham{$1} = $file if ($file =~ m/-(\w+)\.log$/);
- }
- unlink "$opt{tmp}/ham.log.$$";
- unlink "$opt{tmp}/spam.log.$$";
- next unless (scalar keys %spam && scalar keys %ham);
- for my $user (sort keys %spam) {
- next unless defined $ham{$user};
- chdir "$opt{tree}/masses";
- system("cat $opt{corpus}/$ham{$user} >> $opt{tmp}/ham.log.$$");
- system("cat $opt{corpus}/$spam{$user} >> $opt{tmp}/spam.log.$$");
- open(IN, "./hit-frequencies -xpa $flags $opt{corpus}/$spam{$user} $opt{corpus}/$ham{$user} |");
- while(<IN>) {
- chomp;
- push @output, "$_:$user\n";
- }
- close(IN);
- }
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- push @output, $_;
- }
- close(IN);
- for (sort sort_all @output) {
- print OUT $_;
- }
- }
- elsif ($age eq "age") {
- my @output;
-
- for my $which (("0-1", "1-3", "3-6")) {
- my ($after, $before) = split(/-/, $which);
- # get and filter logs
- chdir $opt{corpus};
- for my $type (("ham", "spam")) {
- open(TMP, "> $opt{tmp}/$type.log.$$");
- my @array = ($type eq "ham") ? @ham : @spam;
- for my $file (@array) {
- open(IN, $file);
- while (<IN>) {
- print TMP $_ if time_filter($after, $before);
- }
- close(IN);
- }
- close (TMP);
- }
- # print out by age
- chdir "$opt{tree}/masses";
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- chomp;
- push @output, "$_:$which\n";
- }
- close(IN);
- }
- for (sort sort_all @output) {
- print OUT $_;
- }
- }
- elsif (@ham && @spam) {
- # get logs
- system("cat " . join(" ", @ham) . " > $opt{tmp}/ham.log.$$");
- system("cat " . join(" ", @spam) . " > $opt{tmp}/spam.log.$$");
-
- chdir "$opt{tree}/masses";
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- print(OUT);
- }
- close(IN);
- }
- close(OUT);
- }
+ else {
+ @spam = grep { $_ ne $nightly } @spam;
+ }
+ }
+ }
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+ }
+
+ # age
+ if ($class eq "NET" && $age ne "7day") {
+ @ham = grep { -M "$_" < 10 } @ham;
+ @spam = grep { -M "$_" < 10 } @spam;
+ # find most recent CVS revision
+ my $wanted = 0.0;
+ for (@spam, @ham) {
+ $wanted = $revision{$_} if ($revision{$_} > $wanted);
+ }
+ @spam = grep { $revision{$_} eq $wanted } @spam;
+ @ham = grep { $revision{$_} eq $wanted } @ham;
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+ }
+ elsif ($age =~ /^(?:new|all|age)$/) {
+ @ham = grep { -M "$_" < -M $opt{tagtime} } @ham;
+ @spam = grep { -M "$_" < -M $opt{tagtime} } @spam;
+ if (!$opt{ignore_revisions}) {
+ @ham = grep { $revision{$_} eq $revision } @ham;
+ @spam = grep { $revision{$_} eq $revision } @spam;
+ }
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+ }
+ elsif ($age =~ /(\d+)day/) {
+ my $mtime = $1;
+ @ham = grep { -M "$_" < $mtime } @ham;
+ @spam = grep { -M "$_" < $mtime } @spam;
+ print STDERR "ham: " . join(' ', @ham) . "\n";
+ print STDERR "spam: " . join(' ', @spam) . "\n";
+ }
+
+ open(OUT, "> $opt{html}/$class.$age");
+ print OUT "# ham results used: " . join(" ", @ham) . "\n";
+ print OUT "# spam results used: " . join(" ", @spam) . "\n";
+ for (@ham) {
+ print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
+ }
+ for (@spam) {
+ print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
+ }
+
+ my $flags = "";
+ $flags = "-t net -s 1" if $class eq "NET";
+ $flags = "-M HTML_MESSAGE" if $class eq "HTML";
+ $flags = "-o" if $class eq "OVERLAP";
+
+ if ($age eq "all") {
+ my %spam;
+ my %ham;
+ my @output;
+
+ for my $file (@spam) {
+ $spam{$1} = $file if ($file =~ m/-(\w+)\.log$/);
+ }
+ for my $file (@ham) {
+ $ham{$1} = $file if ($file =~ m/-(\w+)\.log$/);
+ }
+ unlink "$opt{tmp}/ham.log.$$";
+ unlink "$opt{tmp}/spam.log.$$";
+ next unless (scalar keys %spam && scalar keys %ham);
+ for my $user (sort keys %spam) {
+ next unless defined $ham{$user};
+ chdir "$opt{tree}/masses";
+ system("cat $opt{corpus}/$ham{$user} >> $opt{tmp}/ham.log.$$");
+ system("cat $opt{corpus}/$spam{$user} >> $opt{tmp}/spam.log.$$");
+ open(IN, "./hit-frequencies -xpa $flags $opt{corpus}/$spam{$user} $opt{corpus}/$ham{$user} |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$user\n";
+ }
+ close(IN);
+ }
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ push @output, $_;
+ }
+ close(IN);
+ for (sort sort_all @output) {
+ print OUT $_;
+ }
+ }
+ elsif ($age eq "age") {
+ my @output;
+
+ for my $which (("0-1", "1-3", "3-6")) {
+ my ($after, $before) = split(/-/, $which);
+ # get and filter logs
+ chdir $opt{corpus};
+ for my $type (("ham", "spam")) {
+ open(TMP, "> $opt{tmp}/$type.log.$$");
+ my @array = ($type eq "ham") ? @ham : @spam;
+ for my $file (@array) {
+ open(IN, $file);
+ while (<IN>) {
+ print TMP $_ if time_filter($after, $before);
+ }
+ close(IN);
+ }
+ close (TMP);
+ }
+ # print out by age
+ chdir "$opt{tree}/masses";
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$which\n";
+ }
+ close(IN);
+ }
+ for (sort sort_all @output) {
+ print OUT $_;
+ }
+ }
+ elsif (@ham && @spam) {
+ # get logs
+ system("cat " . join(" ", @ham) . " > $opt{tmp}/ham.log.$$");
+ system("cat " . join(" ", @spam) . " > $opt{tmp}/spam.log.$$");
+
+ chdir "$opt{tree}/masses";
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ print(OUT);
+ }
+ close(IN);
}
+ close(OUT);
}