You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2005/10/14 04:42:34 UTC
svn commit: r320979 - /spamassassin/trunk/masses/rule-qa/corpus-hourly
Author: jm
Date: Thu Oct 13 19:42:33 2005
New Revision: 320979
URL: http://svn.apache.org/viewcvs?rev=320979&view=rev
Log:
major changes to hourly corpus scripts; use SVN revisions and dates to differentiate mass-checks, support ruleqa.cgi script, and upcoming changes for rules-project preflight continuous-mass-check buildbot
Modified:
spamassassin/trunk/masses/rule-qa/corpus-hourly
Modified: spamassassin/trunk/masses/rule-qa/corpus-hourly
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/masses/rule-qa/corpus-hourly?rev=320979&r1=320978&r2=320979&view=diff
==============================================================================
--- spamassassin/trunk/masses/rule-qa/corpus-hourly (original)
+++ spamassassin/trunk/masses/rule-qa/corpus-hourly Thu Oct 13 19:42:33 2005
@@ -3,368 +3,430 @@
# settings are located in $HOME/.corpus
use strict;
-use POSIX qw(nice);
-use constant MONTH => 60*60*24*30;
+use File::Path;
+use POSIX qw(nice strftime);
+use constant WEEK => 60*60*24;
nice(15);
my $configuration = "$ENV{HOME}/.corpus";
my %opt;
-my $revision = "unknown";
-my %revision;
+# my $revision = "unknown";
+my %revision = ();
+my %logs_by_rev = ();
+my %is_net_revision = ();
+my %time = ();
my @files;
my $skip = '';
my $time_start = time;
&configure;
-&version;
+# &version;
&init;
&update;
&locate;
-&rename;
+# &rename;
¤t;
&clean_up;
-sub version {
- my $line;
- if (open(TESTING, "$opt{tree}/rules/70_testing.cf")) {
- chomp($line = <TESTING>);
- if ($line =~ m/^#.*Rev(?:ision)?:\s*(\S+).*/) {
- $revision = $1;
- }
- close(TESTING);
- }
-}
+# sub version {
+# my $line;
+# if (open(TESTING, "$opt{tree}/rules/70_testing.cf")) {
+# chomp($line = <TESTING>);
+# if ($line =~ m/^#.*Rev(?:ision)?:\s*(\S+).*/) {
+# $revision = $1;
+# }
+# close(TESTING);
+# }
+# }
sub configure {
- # does rough equivalent of source
- open(C, $configuration) || die "open failed: $configuration: $!\n";
- while(<C>) {
+ # does rough equivalent of source
+ open(C, $configuration) || die "open failed: $configuration: $!\n";
+ while(<C>) {
chomp;
s/#.*//;
if (/^\s*(.*?)\s*=\s*(.*?)\s*$/) {
- $opt{$1} = $2;
+ $opt{$1} = $2;
}
- }
- close(C);
+ }
+ close(C);
}
sub clean_up {
- system "rm -f $opt{tmp}/*.$$";
+ system "rm -f $opt{tmp}/*.$$";
}
sub init {
- $SIG{INT} = \&clean_up;
- $SIG{TERM} = \&clean_up;
+ $SIG{INT} = \&clean_up;
+ $SIG{TERM} = \&clean_up;
- $ENV{RSYNC_PASSWORD} = $opt{password};
- $ENV{TIME} = '%e,%U,%S';
- $ENV{TZ} = 'UTC';
+ $ENV{RSYNC_PASSWORD} = $opt{password};
+ $ENV{TIME} = '%e,%U,%S';
+ $ENV{TZ} = 'UTC';
}
sub update {
- chdir $opt{corpus};
+ chdir $opt{corpus};
- # allow non-running of rsync under some circumstances
- if ($opt{rsync_command}) {
- system $opt{rsync_command};
- } else {
- system "rsync -CPcvuzt --timeout=300 $opt{username}" . '@rsync.spamassassin.org::corpus/*.log .';
- }
+ # allow non-running of rsync under some circumstances
+ if ($opt{rsync_command}) {
+ system $opt{rsync_command};
+ } else {
+ system "rsync -CPcvuzt --timeout=300 $opt{username}" . '@rsync.spamassassin.org::corpus/*.log .';
+ }
+ if (!$opt{always_update_html}) {
if (-f "rsync.last") {
- open(FIND, "find . -type f -newer rsync.last |");
- my $files = "";
- while(<FIND>) {
- $files .= $_;
- }
- close(FIND);
- if (! $files) {
- print STDERR "no new corpus files\n";
- if (!$opt{always_update_html} && rand(24) > 1) {
- exit 0;
- }
- else {
- print STDERR "updating anyway\n";
- }
- }
- }
- open(RSYNC, "> rsync.last");
- close(RSYNC);
- system "chmod +r *.log";
+ open(FIND, "find . -type f -newer rsync.last |");
+ my $files = "";
+ while(<FIND>) {
+ $files .= $_;
+ }
+ close(FIND);
+ if (! $files) {
+ print STDERR "no new corpus files\n";
+ if (rand(24) > 1) {
+ exit 0;
+ }
+ else {
+ print STDERR "updating anyway\n";
+ }
+ }
+ }
+ }
+ open(RSYNC, "> rsync.last");
+ close(RSYNC);
+ system "chmod +r *.log";
}
sub locate {
- chdir "$opt{tree}/masses";
- opendir(CORPUS, $opt{corpus});
- @files = sort readdir(CORPUS);
- closedir(CORPUS);
-
- @files = grep { /^(?:spam|ham)-(?:net-)?\w+\.log$/ && -f "$opt{corpus}/$_" && -M _ < 10 } @files;
- @files = grep {
- my $time = 0;
- my $tag = 0;
- $revision{$_} = "unknown";
- open(FILE, "$opt{corpus}/$_");
- while (my $line = <FILE>) {
- last if $line !~ /^#/;
- $time++ if $line =~ /\b(?!08)\d\d:\d\d:\d\d\b/;
- $revision{$_} = $1 if $line =~ m/(?:CVS|SVN) revision:\s*(\S+)/;
- }
- close(FILE);
- if (!$time) {
- $skip .= "# skipped $_: time is between 0800 UTC and 0900 UTC\n";
- }
- $time;
- } @files;
+ chdir "$opt{tree}/masses";
+ opendir(CORPUS, $opt{corpus});
+ @files = sort readdir(CORPUS);
+ closedir(CORPUS);
+
+ @files = grep {
+ /^(?:spam|ham)-(?:net-)?\w+\.log$/ && -f "$opt{corpus}/$_" && -M _ < 10
+ } @files;
+
+ foreach my $file (@files) {
+ # my $time = 0;
+ my $tag = 0;
+ open(FILE, "$opt{corpus}/$file") or warn "cannot read $opt{corpus}/$file";
+ while (my $line = <FILE>) {
+ last if $line !~ /^#/;
+ if ($line =~ /, on (... ... .. )(..)(:..:.. ... ....)/) {
+ my ($datepre, $hh, $datepost) = ($1,$2,$3);
+
+ use Time::ParseDate;
+ my $timet = Time::ParseDate::parsedate($datepre.$hh.$datepost,
+ GMT => 1, PREFER_PAST => 1);
+ $time{$file} = $timet;
+ print "$opt{corpus}/$file: time=$timet\n";
+
+ # if ($hh != 8) { $time++; }
+ }
+ if ($line =~ m/^# SVN revision:\s*(\S+)/) {
+ my $rev = $1;
+ $revision{$file} = $rev;
+
+ $logs_by_rev{$rev} ||= [ ];
+ push (@{$logs_by_rev{$rev}}, $file);
+
+ if ($file =~ /-net-/) {
+ $is_net_revision{$rev} = 1;
+ print "$opt{corpus}/$file: rev=$rev (net)\n";
+ }
+ else {
+ print "$opt{corpus}/$file: rev=$rev (non-net)\n";
+ }
+ }
+ }
+ close(FILE);
+ # if (!$time) {
+ # $skip .= "# skipped $_: time is between 0800 UTC and 0900 UTC\n";
+ # }
+ }
}
sub rename {
- use File::Copy qw(move);
+ use File::Copy qw(move);
- my $hour = (gmtime($time_start))[2];
- if ($hour == 9) {
+ my $hour = (gmtime($time_start))[2];
+ if ($hour == 9) {
chdir $opt{html};
opendir(HTML, $opt{html});
my @html = readdir(HTML);
closedir(HTML);
@html = grep { -f } @html;
for (@html) {
- move($_, "last/$_");
+ move($_, "last/$_");
}
- }
+ }
}
sub sort_all {
- my ($a1, $a2) = ($a =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
- my ($b1, $b2) = ($b =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
+ my ($a1, $a2) = ($a =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
+ my ($b1, $b2) = ($b =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
- my $n = ($a1 cmp $b1) || (($a2 || '') cmp ($b2 || ''));
- if ($a1 =~ /^OVERALL/) { $n -= 1000; }
- elsif ($a1 =~ /^\(all messages\)/) { $n -= 100; }
- elsif ($a1 =~ /^\(all messages as \%\)/) { $n -= 10; }
- if ($b1 =~ /^OVERALL/) { $n += 1000; }
- elsif ($b1 =~ /^\(all messages\)/) { $n += 100; }
- elsif ($b1 =~ /^\(all messages as \%\)/) { $n += 10; }
- return $n;
+ my $n = ($a1 cmp $b1) || (($a2 || '') cmp ($b2 || ''));
+ if ($a1 =~ /^OVERALL/) { $n -= 1000; }
+ elsif ($a1 =~ /^\(all messages\)/) { $n -= 100; }
+ elsif ($a1 =~ /^\(all messages as \%\)/) { $n -= 10; }
+ if ($b1 =~ /^OVERALL/) { $n += 1000; }
+ elsif ($b1 =~ /^\(all messages\)/) { $n += 100; }
+ elsif ($b1 =~ /^\(all messages as \%\)/) { $n += 10; }
+ return $n;
}
sub time_filter {
- my ($after, $before) = @_;
- if (/time=(\d+)/) {
- return (($time_start - $1 >= MONTH * $after) &&
- ($time_start - $1 < MONTH * $before));
- }
- return 0;
+ my ($after, $before) = @_;
+ if (/time=(\d+)/) {
+ return (($time_start - $1 >= WEEK * $after) &&
+ ($time_start - $1 < WEEK * $before));
+ }
+ return 0;
}
sub current {
- my $classes = $opt{output_classes};
- $classes ||= "DETAILS.new DETAILS.all DETAILS.age HTML.new HTML.all HTML.age NET.new NET.all NET.age";
+ my $classes = $opt{output_classes};
+ $classes ||= "DETAILS.new DETAILS.all DETAILS.age HTML.new HTML.all HTML.age NET.new NET.all NET.age";
+
+ foreach my $entry (split(' ', $classes)) {
+ $entry =~ /^(\S+)\.(\S+)$/;
+ my $class = $1;
+ my $age = $2;
+ if (!$age) { warn "no age in $entry"; next; }
+
+ foreach my $rev (sort keys %logs_by_rev) {
+ next if ($rev eq 'unknown');
+
+ if ($class =~ /NET/) {
+ next unless $is_net_revision{$rev};
+ }
- foreach my $entry (split(' ', $classes)) {
- $entry =~ /^(\S+)\.(\S+)$/;
- my $class = $1;
- my $age = $2;
- if (!$age) { warn "no age in $entry"; next; }
- gen_class ($class, $age);
+ gen_class ($rev, $class, $age);
}
+ }
}
sub gen_class {
- my ($class, $age) = @_;
+ my ($rev, $class, $age) = @_;
- print STDERR "generating $class.$age\n";
+ print STDERR "\ngenerating r$rev $class.$age:\n";
- next if ($class eq "NET" && $age !~ /^(?:new|all|age|7day)$/);
+ next if ($class eq "NET" && $age !~ /^(?:new|all|age|7day)$/);
- my @ham = grep { /^ham/ } @files;
- my @spam = grep { /^spam/ } @files;
+ my @ham = grep { /^ham/ } @{$logs_by_rev{$rev}};
+ my @spam = grep { /^spam/ } @{$logs_by_rev{$rev}};
- print STDERR "ham: " . join(' ', @ham) . "\n";
- print STDERR "spam: " . join(' ', @spam) . "\n";
+ print STDERR "input h: " . join(' ', @ham) . "\n";
+ print STDERR "input s: " . join(' ', @spam) . "\n";
- chdir $opt{corpus};
+ chdir $opt{corpus};
- # net vs. local
- if ($class eq "NET") {
- @ham = grep { /-net-/ } @ham;
- @spam = grep { /-net-/ } @spam;
- print STDERR "ham (NET): " . join(' ', @ham) . "\n";
- print STDERR "spam (NET): " . join(' ', @spam) . "\n";
+ # net vs. local
+ if ($class eq "NET") {
+ @ham = grep { /-net-/ } @ham;
+ @spam = grep { /-net-/ } @spam;
+ }
+ else {
+ # if both net and local exist, use newer
+ my %spam;
+ my %ham;
+
+ for my $file (@spam) {
+ $spam{$1}++ if ($file =~ m/-(\w+)\.log$/);
}
- else {
- # if both net and local exist, use newer
- my %spam;
- my %ham;
-
- for my $file (@spam) {
- $spam{$1}++ if ($file =~ m/-(\w+)\.log$/);
- }
- for my $file (@ham) {
- $ham{$1}++ if ($file =~ m/-(\w+)\.log$/);
- }
- while (my ($user, $count) = each %ham) {
- if ($count > 1) {
- my $nightly = "ham-$user.log";
- my $weekly = "ham-net-$user.log";
- if ($revision{$nightly} >= $revision{$weekly}) {
- @ham = grep { $_ ne $weekly } @ham;
- }
- else {
- @ham = grep { $_ ne $nightly } @ham;
- }
- }
- }
- while (my ($user, $count) = each %spam) {
- if ($count > 1) {
- my $nightly = "spam-$user.log";
- my $weekly = "spam-net-$user.log";
- if ($revision{$nightly} >= $revision{$weekly}) {
- @spam = grep { $_ ne $weekly } @spam;
- }
- else {
- @spam = grep { $_ ne $nightly } @spam;
- }
- }
- }
- print STDERR "ham (non-NET): " . join(' ', @ham) . "\n";
- print STDERR "spam (non-NET): " . join(' ', @spam) . "\n";
+ for my $file (@ham) {
+ $ham{$1}++ if ($file =~ m/-(\w+)\.log$/);
}
+ while (my ($user, $count) = each %ham) {
+ if ($count > 1) {
+ my $nightly = "ham-$user.log";
+ my $weekly = "ham-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @ham = grep { $_ ne $weekly } @ham;
+ }
+ else {
+ @ham = grep { $_ ne $nightly } @ham;
+ }
+ }
+ }
+ while (my ($user, $count) = each %spam) {
+ if ($count > 1) {
+ my $nightly = "spam-$user.log";
+ my $weekly = "spam-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @spam = grep { $_ ne $weekly } @spam;
+ }
+ else {
+ @spam = grep { $_ ne $nightly } @spam;
+ }
+ }
+ }
+ }
+
+ # age
+ if ($age =~ /(\d+)day/) {
+ my $mtime = $1;
+ @ham = grep { -M "$_" < $mtime } @ham;
+ @spam = grep { -M "$_" < $mtime } @spam;
+ }
+ elsif ($class ne 'NET' && $age =~ /^(?:new|all|age)$/)
+ {
+ # just ignore the tagtime stuff; since we now may be
+ # dealing with multiple mass-checks per day, just use svn rev data
+ # my $tt = (-M $opt{tagtime});
+ # @ham = grep { !defined($tt) || ((-M "$_") < $tt) } @ham;
+ # @spam = grep { !defined($tt) || ((-M "$_") < $tt) } @spam;
+ }
+
+ print STDERR "selected h: " . join(' ', @ham) . "\n";
+ print STDERR "selected s: " . join(' ', @spam) . "\n";
+
+ # we cannot continue if we have no files that match the criteria...
+ # demand at least 1 ham and 1 spam file
+ if (scalar @spam <= 0 || scalar @ham <= 0) {
+ warn "not enough files found matching criteria ($rev $class $age)\n";
+ return;
+ }
+
+ my $time = $time{$ham[0]}; # use the ham file's time
+ my $dir = create_outputdir($rev, $time);
+
+ my $fname = "$dir/$class.$age";
+
+ # now, if the target file already exists, check to see if it's newer
+ # than all the sources, make-style
+ if (-f $fname) {
+ my $targetfreshness = (-M $fname);
+ my $needsrebuild = 0;
+
+ foreach my $srcfile (@spam, @ham) {
+ my $srcfreshness = (-M $srcfile);
+ if ($targetfreshness > $srcfreshness) { # src is fresher
+ print "$fname is older than $srcfile: $targetfreshness > $srcfreshness\n";
+ $needsrebuild = 1;
+ last;
+ }
+ }
+
+ if (!$needsrebuild) {
+ print "existing: $fname, fresher than sources\n";
+ return;
+ }
+ }
+
+ open(OUT, "> $fname") or warn "cannot write to $fname";
+ print OUT "# ham results used for $rev $class $age: " . join(" ", @ham) . "\n";
+ print OUT "# spam results used for $rev $class $age: " . join(" ", @spam) . "\n";
+ for (@ham) {
+ print OUT "# $_ was at r$revision{$_}\n";
+ }
+ for (@spam) {
+ print OUT "# $_ was at r$revision{$_}\n";
+ }
+
+ print "creating: ${fname}...\n";
+
+ my $flags = "";
+ $flags = "-t net -s 1" if $class eq "NET";
+ $flags = "-M HTML_MESSAGE" if $class eq "HTML";
+ $flags = "-o" if $class eq "OVERLAP";
+
+ if ($age eq "all") {
+ my %spam;
+ my %ham;
+ my @output;
- # age
- if ($class eq "NET" && $age ne "7day") {
- @ham = grep { -M "$_" < 10 } @ham;
- @spam = grep { -M "$_" < 10 } @spam;
- # find most recent CVS revision
- my $wanted = 0.0;
- for (@spam, @ham) {
- $wanted = $revision{$_} if ($revision{$_} > $wanted);
- }
- @spam = grep { $revision{$_} eq $wanted } @spam;
- @ham = grep { $revision{$_} eq $wanted } @ham;
- print STDERR "ham (NET $age): " . join(' ', @ham) . "\n";
- print STDERR "spam (NET $age): " . join(' ', @spam) . "\n";
- }
- elsif ($age =~ /^(?:new|all|age)$/)
- {
- my $tt = (-M $opt{tagtime});
- @ham = grep { !defined($tt) || ((-M "$_") < $tt) } @ham;
- @spam = grep { !defined($tt) || ((-M "$_") < $tt) } @spam;
-
- if (!$opt{ignore_revisions}) {
- @ham = grep { $revision{$_} eq $revision } @ham;
- @spam = grep { $revision{$_} eq $revision } @spam;
- }
- print STDERR "ham (non-NET $age): " . join(' ', @ham) . "\n";
- print STDERR "spam (non-NET $age): " . join(' ', @spam) . "\n";
+ for my $file (@spam) {
+ $spam{$1} = $file if ($file =~ m/-(\w+)\.log$/);
}
- elsif ($age =~ /(\d+)day/) {
- my $mtime = $1;
- @ham = grep { -M "$_" < $mtime } @ham;
- @spam = grep { -M "$_" < $mtime } @spam;
- print STDERR "ham (non-NET $age): " . join(' ', @ham) . "\n";
- print STDERR "spam (non-NET $age): " . join(' ', @spam) . "\n";
+ for my $file (@ham) {
+ $ham{$1} = $file if ($file =~ m/-(\w+)\.log$/);
}
-
- open(OUT, "> $opt{html}/$class.$age");
- print OUT "# ham results used: " . join(" ", @ham) . "\n";
- print OUT "# spam results used: " . join(" ", @spam) . "\n";
- for (@ham) {
- print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
- }
- for (@spam) {
- print OUT "# $_=$revision{$_}\n" if $revision{$_} ne $revision;
- }
-
- my $flags = "";
- $flags = "-t net -s 1" if $class eq "NET";
- $flags = "-M HTML_MESSAGE" if $class eq "HTML";
- $flags = "-o" if $class eq "OVERLAP";
-
- if ($age eq "all") {
- my %spam;
- my %ham;
- my @output;
-
- for my $file (@spam) {
- $spam{$1} = $file if ($file =~ m/-(\w+)\.log$/);
- }
- for my $file (@ham) {
- $ham{$1} = $file if ($file =~ m/-(\w+)\.log$/);
- }
- unlink "$opt{tmp}/ham.log.$$";
- unlink "$opt{tmp}/spam.log.$$";
-
- if (!(scalar keys %spam && scalar keys %ham)) {
- warn "no files found for $class.$age";
- return;
- }
+ unlink "$opt{tmp}/ham.log.$$";
+ unlink "$opt{tmp}/spam.log.$$";
- for my $user (sort keys %spam) {
- next unless defined $ham{$user};
- chdir "$opt{tree}/masses";
- system("cat $opt{corpus}/$ham{$user} >> $opt{tmp}/ham.log.$$");
- system("cat $opt{corpus}/$spam{$user} >> $opt{tmp}/spam.log.$$");
- open(IN, "./hit-frequencies -xpa $flags $opt{corpus}/$spam{$user} $opt{corpus}/$ham{$user} |");
- while(<IN>) {
- chomp;
- push @output, "$_:$user\n";
- }
- close(IN);
- }
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- push @output, $_;
- }
- close(IN);
- for (sort sort_all @output) {
- print OUT $_;
- }
+ if (scalar keys %spam <= 0 || scalar keys %ham <= 0) {
+ warn "no files found for $class.$age";
+ return;
}
- elsif ($age eq "age") {
- my @output;
- for my $which (("0-1", "1-3", "3-6")) {
- my ($after, $before) = split(/-/, $which);
- # get and filter logs
- chdir $opt{corpus};
- for my $type (("ham", "spam")) {
- open(TMP, "> $opt{tmp}/$type.log.$$");
- my @array = ($type eq "ham") ? @ham : @spam;
- for my $file (@array) {
- open(IN, $file);
- while (<IN>) {
- print TMP $_ if time_filter($after, $before);
- }
- close(IN);
- }
- close (TMP);
- }
- # print out by age
- chdir "$opt{tree}/masses";
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- chomp;
- push @output, "$_:$which\n";
- }
- close(IN);
- }
- for (sort sort_all @output) {
- print OUT $_;
- }
- }
- elsif (@ham && @spam) {
- # get logs
- system("cat " . join(" ", @ham) . " > $opt{tmp}/ham.log.$$");
- system("cat " . join(" ", @spam) . " > $opt{tmp}/spam.log.$$");
-
- chdir "$opt{tree}/masses";
- open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
- while(<IN>) {
- print(OUT);
- }
- close(IN);
- }
- close(OUT);
+ chdir "$opt{tree}/masses";
+ for my $user (sort keys %spam) {
+ next unless $ham{$user};
+ system("cat $opt{corpus}/$ham{$user} >> $opt{tmp}/ham.log.$$");
+ system("cat $opt{corpus}/$spam{$user} >> $opt{tmp}/spam.log.$$");
+ open(IN, "./hit-frequencies -xpa $flags $opt{corpus}/$spam{$user} $opt{corpus}/$ham{$user} |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$user\n";
+ }
+ close(IN);
+ }
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ push @output, $_;
+ }
+ close(IN);
+ for (sort sort_all @output) { print OUT; }
+ }
+ elsif ($age eq "age") {
+ my @output;
+
+ for my $which (("0-1", "1-2", "2-3", "3-6")) {
+ my ($after, $before) = split(/-/, $which);
+ # get and filter logs
+ chdir $opt{corpus};
+ for my $type (("ham", "spam")) {
+ open(TMP, "> $opt{tmp}/$type.log.$$");
+ my @array = ($type eq "ham") ? @ham : @spam;
+ for my $file (@array) {
+ open(IN, $file) or warn "cannot read $file";
+ while (<IN>) {
+ print TMP $_ if time_filter($after, $before);
+ }
+ close(IN);
+ }
+ close (TMP);
+ }
+ # print out by age
+ chdir "$opt{tree}/masses";
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$which\n";
+ }
+ close(IN);
+ }
+ for (sort sort_all @output) { print OUT; }
+ }
+ elsif (@ham && @spam) {
+ # get logs
+ system("cat " . join(" ", @ham) . " > $opt{tmp}/ham.log.$$");
+ system("cat " . join(" ", @spam) . " > $opt{tmp}/spam.log.$$");
+
+ chdir "$opt{tree}/masses";
+ open(IN, "./hit-frequencies -xpa $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) { print(OUT); }
+ close(IN);
+ }
+
+ my $bytes = (-s OUT);
+ print "created: $fname, $bytes bytes\n";
+ close(OUT);
+}
+
+sub create_outputdir {
+ my ($rev, $time) = @_;
+ my $dir = $opt{html} .'/'. strftime("%Y%m%d", gmtime($time)) . "/r$rev";
+ # print "output dir: $dir\n";
+ if (!-d $dir) {
+ mkpath($dir) or warn "failed to mkdir $dir";
+ }
+ return $dir;
}
+