You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by da...@apache.org on 2017/06/04 20:29:33 UTC
svn commit: r1797609 - in /spamassassin/trunk/masses/rule-qa:
automc/ruleqa.cgi corpus-hourly
Author: davej
Date: Sun Jun 4 20:29:33 2017
New Revision: 1797609
URL: http://svn.apache.org/viewvc?rev=1797609&view=rev
Log:
Adjusted paths for new server based on new findings for the ruleqa.cgi web UI.
Added:
spamassassin/trunk/masses/rule-qa/corpus-hourly (with props)
Modified:
spamassassin/trunk/masses/rule-qa/automc/ruleqa.cgi
Modified: spamassassin/trunk/masses/rule-qa/automc/ruleqa.cgi
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/rule-qa/automc/ruleqa.cgi?rev=1797609&r1=1797608&r2=1797609&view=diff
==============================================================================
--- spamassassin/trunk/masses/rule-qa/automc/ruleqa.cgi (original)
+++ spamassassin/trunk/masses/rule-qa/automc/ruleqa.cgi Sun Jun 4 20:29:33 2017
@@ -1,5 +1,5 @@
#!/usr/bin/perl
-my $automcdir = "/usr/local/spamassassin/automc/svn/rule-qa/automc";
+my $automcdir = "/usr/local/spamassassin/automc/svn/masses/rule-qa/automc";
###!/usr/bin/perl
##my $automcdir = "/home/jm/ftp/spamassassin/masses/rule-qa/automc";
Added: spamassassin/trunk/masses/rule-qa/corpus-hourly
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/rule-qa/corpus-hourly?rev=1797609&view=auto
==============================================================================
--- spamassassin/trunk/masses/rule-qa/corpus-hourly (added)
+++ spamassassin/trunk/masses/rule-qa/corpus-hourly Sun Jun 4 20:29:33 2017
@@ -0,0 +1,490 @@
+#!/usr/bin/perl -w
+
+# settings are located in $HOME/.corpus
+
+use strict;
+use Getopt::Long;
+
+use vars qw(
+ $corpusdir
+ $opt_override
+ $opt_tag
+);
+GetOptions(
+ "tag=s" => \$opt_tag,
+ "dir=s" => \$corpusdir,
+ "override=s" => \$opt_override,
+);
+
+$opt_override ||= '';
+$opt_tag ||= 'n'; # nightly is the default
+
+
+use File::Path;
+use File::Copy;
+use Time::ParseDate;
+use Cwd;
+use POSIX qw(nice strftime);
+
+use constant WEEK => 60*60*24;
+nice(15);
+
+my $configuration = "$ENV{HOME}/.corpus";
+my %opt;
+my %revision = ();
+my %logs_by_rev = ();
+my %is_net_revision = ();
+my %time = ();
+my @files;
+my @tmps = ();
+my $skip = '';
+my $time_start = time;
+my $output_revpath;
+
+&configure;
+&init;
+
+if ($corpusdir) {
+ print "reading logs from '$corpusdir'\n";
+}
+else {
+ $corpusdir = $opt{corpus};
+ &update_rsync;
+}
+
+&locate;
+¤t;
+&clean_up;
+
+sub configure {
+ # does rough equivalent of source
+ open(C, $configuration) || die "open failed: $configuration: $!\n";
+ my $pwd = getcwd;
+
+ # add 'override' options
+ my @lines = (<C>, split(/\|/, $opt_override));
+
+ foreach $_ (@lines) {
+ chomp;
+ s/#.*//;
+ if (/^\s*(.*?)\s*=\s*(.*?)\s*$/) {
+ my ($key, $val) = ($1, $2);
+ $val =~ s/\$PWD/$pwd/gs;
+ $opt{$key} = $val;
+ }
+ }
+ close(C);
+}
+
+sub clean_up {
+ system "rm -f $opt{tmp}/*.$$ ".join(' ', @tmps);
+}
+
+sub init {
+ $SIG{INT} = \&clean_up;
+ $SIG{TERM} = \&clean_up;
+
+ $ENV{RSYNC_PASSWORD} = $opt{password};
+ $ENV{TIME} = '%e,%U,%S';
+ $ENV{TZ} = 'UTC';
+}
+
+sub update_rsync {
+ chdir $corpusdir;
+
+ # allow non-running of rsync under some circumstances
+ if ($opt{rsync_command}) {
+ system $opt{rsync_command};
+ } else {
+ system "rsync -CPcvuzt --timeout=300 $opt{username}" . '@rsync.spamassassin.org::corpus/*.log .';
+ }
+
+ # this block is no longer required -- we do sensible things with modtime
+ # comparisons to work it out!
+ if (0 && !$opt{always_update_html}) {
+ if (-f "rsync.last") {
+ open(FIND, "find . -type f -newer rsync.last |");
+ my $files = "";
+ while(<FIND>) {
+ $files .= $_;
+ }
+ close(FIND);
+ if (! $files) {
+ print STDERR "no new corpus files\n";
+ if (rand(24) > 1) {
+ exit 0;
+ }
+ else {
+ print STDERR "updating anyway\n";
+ }
+ }
+ }
+ }
+
+ open(RSYNC, "> rsync.last");
+ close(RSYNC);
+ system "chmod +r *.log";
+}
+
+sub locate {
+ # chdir "$opt{tree}/masses" or die "cannot chdir $opt{tree}/masses";
+
+ opendir(CORPUS, $corpusdir);
+ @files = sort readdir(CORPUS);
+ closedir(CORPUS);
+
+ @files = grep {
+ /^(?:spam|ham)-(?:net-)?[-\w]+\.log$/ && -f "$corpusdir/$_" && -M _ < 10
+ } @files;
+
+ foreach my $file (@files) {
+ # my $time = 0;
+ my $tag = 0;
+ open(FILE, "$corpusdir/$file") or warn "cannot read $corpusdir/$file";
+ while (my $line = <FILE>) {
+ last if $line !~ /^#/;
+ if ($line =~ /, on (... ... .. )(..)(:..:.. ... ....)/) {
+ my ($datepre, $hh, $datepost) = ($1,$2,$3);
+
+ my $timet = Time::ParseDate::parsedate($datepre.$hh.$datepost,
+ GMT => 1, PREFER_PAST => 1);
+ $time{$file} = $timet;
+ print "$corpusdir/$file: time=$timet\n";
+
+ # if ($hh != 8) { $time++; }
+ }
+ if ($line =~ m/^# SVN revision:\s*(\S+)/) {
+ my $rev = $1;
+ $revision{$file} = $rev;
+
+ $logs_by_rev{$rev} ||= [ ];
+ push (@{$logs_by_rev{$rev}}, $file);
+
+ if ($file =~ /-net-/) {
+ $is_net_revision{$rev} = 1;
+ print "$corpusdir/$file: rev=$rev (net)\n";
+ }
+ else {
+ print "$corpusdir/$file: rev=$rev (non-net)\n";
+ }
+ }
+ }
+ close(FILE);
+ # if (!$time) {
+ # $skip .= "# skipped $_: time is between 0800 UTC and 0900 UTC\n";
+ # }
+ }
+}
+
+sub sort_all {
+ my ($a1, $a2) = ($a =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
+ my ($b1, $b2) = ($b =~ m/(\(.*?\)|\S+)(?::(\S+))?$/);
+ $a1 =~ s/^[\+\-]//;
+ $b1 =~ s/^[\+\-]//;
+
+ my $n = ($a1 cmp $b1) || (($a2 || '') cmp ($b2 || ''));
+ if ($a1 =~ /^OVERALL/) { $n -= 1000; }
+ elsif ($a1 =~ /^\(all messages\)/) { $n -= 100; }
+ elsif ($a1 =~ /^\(all messages as \%\)/) { $n -= 10; }
+ if ($b1 =~ /^OVERALL/) { $n += 1000; }
+ elsif ($b1 =~ /^\(all messages\)/) { $n += 100; }
+ elsif ($b1 =~ /^\(all messages as \%\)/) { $n += 10; }
+ return $n;
+}
+
+sub time_filter {
+ my ($after, $before) = @_;
+ if (/time=(\d+)/) {
+ return (($time_start - $1 >= WEEK * $after) &&
+ ($time_start - $1 < WEEK * $before));
+ }
+ return 0;
+}
+
+sub current {
+ my $classes = $opt{output_classes};
+ $classes ||= "DETAILS.new DETAILS.all DETAILS.age HTML.new HTML.all HTML.age NET.new NET.all NET.age";
+
+ foreach my $entry (split(' ', $classes)) {
+ $entry =~ /^(\S+)\.(\S+)$/;
+ my $class = $1;
+ my $age = $2;
+ if (!$age) { warn "no age in $entry"; next; }
+
+ foreach my $rev (sort keys %logs_by_rev) {
+ next if ($rev eq 'unknown');
+
+ if ($class =~ /NET/) {
+ next unless $is_net_revision{$rev};
+ }
+
+ gen_class ($rev, $class, $age);
+ }
+ }
+}
+
+sub gen_class {
+ my ($rev, $class, $age) = @_;
+
+ print STDERR "\ngenerating r$rev $class.$age:\n";
+
+ next if ($class eq "NET" && $age !~ /^(?:new|all|age|7day)$/);
+
+ my @ham = grep { /^ham/ } @{$logs_by_rev{$rev}};
+ my @spam = grep { /^spam/ } @{$logs_by_rev{$rev}};
+
+ print STDERR "input h: " . join(' ', @ham) . "\n";
+ print STDERR "input s: " . join(' ', @spam) . "\n";
+
+ chdir $corpusdir;
+
+ # net vs. local
+ if ($class eq "NET") {
+ @ham = grep { /-net-/ } @ham;
+ @spam = grep { /-net-/ } @spam;
+ }
+ else {
+ # if both net and local exist, use newer
+ my %spam;
+ my %ham;
+
+ for my $file (@spam) {
+ $spam{$1}++ if ($file =~ m/-(\w[-\w]+)\.log$/);
+ }
+ for my $file (@ham) {
+ $ham{$1}++ if ($file =~ m/-(\w[-\w]+)\.log$/);
+ }
+ while (my ($user, $count) = each %ham) {
+ if ($count > 1) {
+ my $nightly = "ham-$user.log";
+ my $weekly = "ham-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @ham = grep { $_ ne $weekly } @ham;
+ }
+ else {
+ @ham = grep { $_ ne $nightly } @ham;
+ }
+ }
+ }
+ while (my ($user, $count) = each %spam) {
+ if ($count > 1) {
+ my $nightly = "spam-$user.log";
+ my $weekly = "spam-net-$user.log";
+ if ($revision{$nightly} >= $revision{$weekly}) {
+ @spam = grep { $_ ne $weekly } @spam;
+ }
+ else {
+ @spam = grep { $_ ne $nightly } @spam;
+ }
+ }
+ }
+ }
+
+ # age
+ if ($age =~ /(\d+)day/) {
+ my $mtime = $1;
+ @ham = grep { -M "$_" < $mtime } @ham;
+ @spam = grep { -M "$_" < $mtime } @spam;
+ }
+ elsif ($class ne 'NET' && $age =~ /^(?:new|all|age)$/)
+ {
+ # just ignore the tagtime stuff; since we now may be
+ # dealing with multiple mass-checks per day, just use svn rev data
+ # my $tt = (-M $opt{tagtime});
+ # @ham = grep { !defined($tt) || ((-M "$_") < $tt) } @ham;
+ # @spam = grep { !defined($tt) || ((-M "$_") < $tt) } @spam;
+ }
+
+ print STDERR "selected h: " . join(' ', @ham) . "\n";
+ print STDERR "selected s: " . join(' ', @spam) . "\n";
+
+ # we cannot continue if we have no files that match the criteria...
+ # demand at least 1 ham and 1 spam file
+ if (scalar @spam <= 0 || scalar @ham <= 0) {
+ warn "not enough files found matching criteria ($rev $class $age)\n";
+ return;
+ }
+
+ my $time = $time{$ham[0]}; # use the ham file's time
+ my $dir = create_outputdir($rev, $time);
+
+ my $fname = "$dir/$class.$age";
+
+ # now, if the target file already exists, check to see if it's newer
+ # than all the sources, make-style
+ if (-f $fname) {
+ my $targetfreshness = (-M $fname);
+ my $needsrebuild = 0;
+
+ foreach my $srcfile (@spam, @ham) {
+ my $srcfreshness = (-M $srcfile);
+ if ($targetfreshness > $srcfreshness) { # src is fresher
+ print "$fname is older than $srcfile: $targetfreshness > $srcfreshness\n";
+ $needsrebuild = 1;
+ last;
+ }
+ }
+
+ if (!$needsrebuild) {
+ print "existing: $fname, fresher than sources\n";
+ return;
+ }
+ }
+
+ my $when = scalar localtime time;
+ print qq{creating: $fname
+ started $when...
+ };
+ my $bytes = 0;
+
+ if ($class eq 'LOGS') {
+ foreach my $f (@ham, @spam) {
+ $f =~ s/[^-\._A-Za-z0-9]+/_/gs; # sanitize!
+ my $zf = "$fname-$f.gz";
+
+ system("gzip -c < $f > $zf.$$");
+ if ($? >> 8 != 0) {
+ warn "gzip -c < $f > $zf.$$ failed";
+ }
+
+ rename("$zf.$$", $zf) or
+ warn "cannot rename $zf.$$ to $zf";
+ $bytes += (-s $zf);
+ }
+ }
+ else {
+ my $tmpfname = "$fname.$$";
+
+ open(OUT, "> $tmpfname") or warn "cannot write to $tmpfname";
+ print OUT "# ham results used for $rev $class $age: " . join(" ", @ham) . "\n";
+ print OUT "# spam results used for $rev $class $age: " . join(" ", @spam) . "\n";
+ for (@ham) {
+ print OUT "# $_ was at r$revision{$_}\n";
+ }
+ for (@spam) {
+ print OUT "# $_ was at r$revision{$_}\n";
+ }
+
+ push (@tmps, $tmpfname);
+
+ my $flags = "";
+ $flags = "-t net -s 1" if $class eq "NET";
+ $flags = "-M HTML_MESSAGE" if $class eq "HTML";
+ $flags = "-o" if $class eq "OVERLAP";
+ $flags = "-S" if $class eq "SCOREMAP";
+ if ($opt{rules_dir}) {
+ $flags .= " -c '$opt{rules_dir}'";
+ }
+
+ if ($age eq "all") {
+ my %spam;
+ my %ham;
+ my @output;
+
+ for my $file (@spam) {
+ $spam{$1} = $file if ($file =~ m/-(\w[-\w]+)\.log$/);
+ }
+ for my $file (@ham) {
+ $ham{$1} = $file if ($file =~ m/-(\w[-\w]+)\.log$/);
+ }
+ unlink "$opt{tmp}/ham.log.$$";
+ unlink "$opt{tmp}/spam.log.$$";
+
+ if (scalar keys %spam <= 0 || scalar keys %ham <= 0) {
+ warn "no files found for $class.$age";
+ return;
+ }
+
+ chdir "$opt{tree}/masses" or die "cannot chdir $opt{tree}/masses";
+ for my $user (sort keys %spam) {
+ next unless $ham{$user};
+ system("cat $corpusdir/$ham{$user} >> $opt{tmp}/ham.log.$$");
+ system("cat $corpusdir/$spam{$user} >> $opt{tmp}/spam.log.$$");
+ open(IN, "./hit-frequencies -TxpaP $flags $corpusdir/$spam{$user} $corpusdir/$ham{$user} |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$user\n";
+ }
+ close(IN);
+ }
+ open(IN, "./hit-frequencies -TxpaP $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ push @output, $_;
+ }
+ close(IN);
+ for (sort sort_all @output) { print OUT; }
+ }
+ elsif ($age eq "age") {
+ my @output;
+
+ for my $which (("0-1", "1-2", "2-3", "3-6")) {
+ my ($after, $before) = split(/-/, $which);
+ # get and filter logs
+ chdir $corpusdir;
+ for my $type (("ham", "spam")) {
+ open(TMP, "> $opt{tmp}/$type.log.$$");
+ my @array = ($type eq "ham") ? @ham : @spam;
+ for my $file (@array) {
+ open(IN, $file) or warn "cannot read $file";
+ while (<IN>) {
+ print TMP $_ if time_filter($after, $before);
+ }
+ close(IN);
+ }
+ close (TMP);
+ }
+ # print out by age
+ chdir "$opt{tree}/masses" or die "cannot chdir $opt{tree}/masses";
+ open(IN, "./hit-frequencies -TxpaP $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) {
+ chomp;
+ push @output, "$_:$which\n";
+ }
+ close(IN);
+ }
+ for (sort sort_all @output) { print OUT; }
+ }
+ elsif (@ham && @spam) {
+ # get logs
+ system("cat " . join(" ", @ham) . " > $opt{tmp}/ham.log.$$");
+ system("cat " . join(" ", @spam) . " > $opt{tmp}/spam.log.$$");
+
+ chdir "$opt{tree}/masses" or die "cannot chdir $opt{tree}/masses";
+ open(IN, "./hit-frequencies -TxpaP $flags $opt{tmp}/spam.log.$$ $opt{tmp}/ham.log.$$ |");
+ while(<IN>) { print(OUT); }
+ close(IN);
+ }
+
+ $bytes = (-s OUT);
+ close(OUT);
+ rename($tmpfname, $fname) or warn "cannot rename $tmpfname to $fname";
+ }
+
+ $when = scalar localtime time;
+ print qq{created: $bytes bytes, finished at $when
+URL:
+
+ http://buildbot.spamassassin.org/ruleqa?daterev=$output_revpath
+
+};
+
+}
+
+sub create_outputdir {
+ my ($rev, $time) = @_;
+ my $revpath = strftime("%Y%m%d", gmtime($time)) . "/r$rev-$opt_tag";
+ my $dir = $opt{html} .'/'. $revpath;
+
+ # print "output dir: $dir\n";
+ if (!-d $dir) {
+ my $prevu = umask 0;
+ mkpath([$dir], 0, oct($opt{html_mode})) or warn "failed to mkdir $dir";
+ umask $prevu;
+ }
+
+ $output_revpath = $revpath; # set the global
+ $output_revpath =~ s/\//-/; # looks nicer
+
+ return $dir;
+}
+
Propchange: spamassassin/trunk/masses/rule-qa/corpus-hourly
------------------------------------------------------------------------------
svn:executable = *