You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2007/08/19 00:55:22 UTC
svn commit: r567334 - in /spamassassin/trunk:
lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm sa-compile.raw
Author: jm
Date: Sat Aug 18 15:55:20 2007
New Revision: 567334
URL: http://svn.apache.org/viewvc?view=rev&rev=567334
Log:
cache base-strings to speed up sa-compile; store them between runs in ~/.spamassassin/sa-compile.cache. relies on Data::Dumper, which is a base module since 5.005
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm
spamassassin/trunk/sa-compile.raw
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm?view=diff&rev=567334&r1=567333&r2=567334
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/BodyRuleBaseExtractor.pm Sat Aug 18 15:55:20 2007
@@ -31,6 +31,8 @@
use Mail::SpamAssassin::Logger;
use Mail::SpamAssassin::Util::Progress;
+use Data::Dumper;
+
use strict;
use warnings;
use bytes;
@@ -103,7 +105,7 @@
my @failed = ();
my $yes = 0;
my $no = 0;
-
+ my $count = 0;
my $start = time;
$self->{main} = $conf->{main}; # for use in extract_hints()
info ("extracting from rules of type $ruletype");
@@ -115,21 +117,45 @@
# require this many chars in a base string for it to be viable
my $min_chars = 3;
- my $count = 0;
my $progress;
-
$self->{show_progress} and $progress = Mail::SpamAssassin::Util::Progress->new({
total => scalar keys %{$rules},
itemtype => 'rules',
});
+ my $cached = { };
+ my $cachefile;
+
+ if ($self->{main}->{bases_cache_dir}) {
+ $cachefile = $self->{main}->{bases_cache_dir}."/rules.$ruletype";
+ $cached = $self->read_cachefile($cachefile);
+ }
+
+NEXT_RULE:
foreach my $name (keys %{$rules}) {
- my $rule = $rules->{$name};
$self->{show_progress} and $progress->update(++$count);
+ my $rule = $rules->{$name};
+ my $cachekey = join "#", $name, $rule;
+
+ my $cent = $cached->{rule_bases}->{$cachekey};
+ if (defined $cent) {
+ if (defined $cent->{g}) {
+ dbg("zoom: YES (cached) $rule");
+ push @good_bases, @{$cent->{g}};
+ $yes++;
+ }
+ else {
+ dbg("zoom: NO (cached) $rule");
+ push @failed, { orig => $rule }; # no need to cache this
+ $no++;
+ }
+ next NEXT_RULE;
+ }
+
# ignore ReplaceTags rules
# TODO: need cleaner way to do this
- next if ($conf->{rules_to_replace}->{$name});
+ goto NO if ($conf->{rules_to_replace}->{$name});
my ($lossy, @bases);
@@ -163,17 +189,26 @@
}
}
+ my @forcache = ();
foreach my $base (@bases) {
next if $subsumed{$base};
push @good_bases, {
base => $base, orig => $rule, name => "$name,[l=$lossy]"
};
+ # *separate* copies for cache -- we modify the @good_bases entry
+ push @forcache, {
+ base => $base, orig => $rule, name => "$name,[l=$lossy]"
+ };
}
+
+ $cached->{rule_bases}->{$cachekey} = { g => \@forcache };
$yes++;
}
else {
+NO:
dbg("zoom: NO $rule");
push @failed, { orig => $rule };
+ $cached->{rule_bases}->{$cachekey} = { };
$no++;
}
}
@@ -230,9 +265,9 @@
$self->{show_progress} and $progress->update(++$count);
my $base1 = $set1->{base};
- my $orig1 = $set1->{orig};
my $name1 = $set1->{name};
next if ($base1 eq '' or $name1 eq '');
+ my $orig1 = $set1->{orig};
$conf->{base_orig}->{$ruletype}->{$name1} = $orig1;
@@ -283,20 +318,18 @@
foreach my $base (keys %bases) {
# uniq the list, since there are probably dup rules listed
- my @list = split (' ', $bases{$base});
- my @uniqed;
-
- {
- my %u=(); @uniqed = grep {defined} map {
- if (exists $u{$_}) { undef; } else { $u{$_}=undef;$_; }
- } @list; undef %u;
+ my %u = ();
+ for my $i (split ' ', $bases{$base}) {
+ next if exists $u{$i}; undef $u{$i};
}
-
- my $key = join ' ', sort @uniqed;
- $conf->{base_string}->{$ruletype}->{$base} = $key;
+ $conf->{base_string}->{$ruletype}->{$base} = join ' ', sort keys %u;
}
$self->{show_progress} and $progress->final();
+ if ($cachefile) {
+ $self->write_cachefile ($cachefile, $cached);
+ }
+
my $elapsed = time - $start;
info ("$ruletype: ".
(scalar keys %{$conf->{base_string}->{$ruletype}}).
@@ -902,5 +935,35 @@
}
###########################################################################
+
+sub read_cachefile {
+ my ($self, $cachefile) = @_;
+ if (open(IN, "<".$cachefile)) {
+ my $str = join("", <IN>);
+ close IN;
+ $str =~ /^(.*)$/s;
+ my $untainted = $1;
+
+ my $VAR1; # Data::Dumper
+ if (eval $untainted) {
+ return $VAR1; # Data::Dumper's naming
+ }
+ }
+ return { };
+}
+
+sub write_cachefile {
+ my ($self, $cachefile, $cached) = @_;
+
+ my $dump = Data::Dumper->new ([ $cached ]);
+ $dump->Deepcopy(1);
+ $dump->Purity(1);
+ $dump->Indent(1);
+ mkdir ($self->{main}->{bases_cache_dir});
+ open (CACHE, ">$cachefile") or warn "cannot write to $cachefile";
+ print CACHE $dump->Dump, ";1;";
+ close CACHE or warn "cannot close $cachefile";
+ warn "JMD $cachefile";
+}
1;
Modified: spamassassin/trunk/sa-compile.raw
URL: http://svn.apache.org/viewvc/spamassassin/trunk/sa-compile.raw?view=diff&rev=567334&r1=567333&r2=567334
==============================================================================
--- spamassassin/trunk/sa-compile.raw (original)
+++ spamassassin/trunk/sa-compile.raw Sat Aug 18 15:55:20 2007
@@ -130,6 +130,7 @@
# ensure the body-rule base extractor plugin is loaded, we use that
my $post_config = q(
loadplugin Mail::SpamAssassin::Plugin::BodyRuleBaseExtractor
+
).join("\n", @{$opt{'cf'}})."\n";
my $spamtest = new Mail::SpamAssassin(
@@ -169,6 +170,9 @@
{
die "sa-compile: cannot write to $installdir, aborting\n";
}
+
+$spamtest->{bases_cache_dir} = $spamtest->get_and_create_userstate_dir()
+ ."/sa-compile.cache";
$spamtest->init(1);
my $conf = $spamtest->{conf};