You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2006/11/12 23:10:22 UTC
svn commit: r474059 - in /spamassassin/trunk: lib/Mail/SpamAssassin.pm
lib/Mail/SpamAssassin/Conf.pm lib/Mail/SpamAssassin/Conf/Parser.pm
lib/Mail/SpamAssassin/PerMsgStatus.pm lib/Mail/SpamAssassin/Plugin/Check.pm
masses/mass-check
Author: jm
Date: Sun Nov 12 14:10:21 2006
New Revision: 474059
URL: http://svn.apache.org/viewvc?view=rev&rev=474059
Log:
reduce memory footprint by about 750KB by: deleting the source for compiled rulesets; deleting stuff used to parse config; compacting the descriptions hash into a single string, for more RAM-efficient but slower lookups
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
spamassassin/trunk/masses/mass-check
Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin.pm Sun Nov 12 14:10:21 2006
@@ -430,7 +430,6 @@
sub check {
my ($self, $mail_obj) = @_;
- local ($_);
$self->init(1);
my $msg = Mail::SpamAssassin::PerMsgStatus->new($self, $mail_obj);
@@ -1227,6 +1226,8 @@
$mail->finish();
$self->finish_learner();
+ $self->{conf}->free_uncompiled_rule_source();
+
# load SQL modules now as well
my $dsn = $self->{conf}->{user_scores_dsn};
if ($dsn ne '') {
@@ -1399,7 +1400,7 @@
}
if ($sysrules) {
- $self->{config_text} .= $self->read_cf ($sysrules, 'default rules dir');
+ $self->{config_text} .= $self->read_cf($sysrules, 'default rules dir');
}
if (!$self->{languages_filename}) {
@@ -1407,7 +1408,7 @@
}
if ($siterules) {
- $self->{config_text} .= $self->read_cf ($siterules, 'site rules dir');
+ $self->{config_text} .= $self->read_cf($siterules, 'site rules dir');
}
if ( $use_user_pref != 0 ) {
@@ -1427,7 +1428,7 @@
}
}
- $self->{config_text} .= $self->read_cf ($fname, 'user prefs file');
+ $self->{config_text} .= $self->read_cf($fname, 'user prefs file');
}
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Sun Nov 12 14:10:21 2006
@@ -2893,6 +2893,17 @@
###########################################################################
+sub get_description_for_rule {
+ my ($self, $rule) = @_;
+ if ($self->{descriptions_str} =~ /^\Q${rule}\E:(.*?)$/m) {
+ return $1;
+ } else {
+ return;
+ }
+}
+
+###########################################################################
+
sub maybe_header_only {
my($self,$rulename) = @_;
my $type = $self->{test_types}->{$rulename};
@@ -3031,6 +3042,15 @@
delete $dest->{sed_path_cache};
return 1;
+}
+
+###########################################################################
+
+sub free_uncompiled_rule_source {
+ my ($self) = @_;
+ delete $self->{if_stack};
+ delete $self->{source_file};
+ delete $self->{meta_dependencies};
}
###########################################################################
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm Sun Nov 12 14:10:21 2006
@@ -663,6 +663,7 @@
$self->trace_meta_dependencies();
$self->fix_priorities();
+ $self->compact_descriptions();
dbg("conf: finish parsing");
@@ -750,8 +751,11 @@
# named this way just in case we ever want a "finish_parsing_start"
$conf->{main}->call_plugins("finish_parsing_end", { conf => $conf });
- delete $conf->{tests}; # free it up
- delete $conf->{priority}; # free it up
+ # free up stuff we no longer need
+ delete $conf->{tests};
+ delete $conf->{priority};
+ delete $conf->{test_types};
+ # delete $conf->{tflags};
}
sub trace_meta_dependencies {
@@ -831,6 +835,24 @@
}
}
}
+}
+
+# compact the {descriptions} hash into a single string; by using
+# a string, quite a lot of RAM is freed up in exchange for slightly
+# slower lookup time
+sub compact_descriptions {
+ my ($self) = @_;
+ my $conf = $self->{conf};
+
+ my $descs = '';
+ my ($k, $v);
+ while (($k, $v) = each %{$conf->{descriptions}})
+ {
+ $descs .= "$k:$v\n";
+ }
+
+ delete $conf->{descriptions};
+ $conf->{descriptions_str} = $descs;
}
###########################################################################
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Sun Nov 12 14:10:21 2006
@@ -2114,7 +2114,7 @@
$params{score} || $self->{conf}->{scores}->{$rule},
$area,
$params{ruletype},
- ($self->{conf}->{descriptions}->{$rule} || $rule));
+ $self->{conf}->get_description_for_rule($rule) || $rule);
}
###########################################################################
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Sun Nov 12 14:10:21 2006
@@ -318,6 +318,7 @@
no strict "subs";
undef &{"${package_name}::_meta_tests_${clean_priority}"};
use strict "subs";
+ $self->free_ruleset_source($pms, 'meta', $priority);
return unless ($evalstr);
@@ -473,6 +474,7 @@
no strict "subs";
undef &{"${package_name}::_head_tests_${clean_priority}"};
use strict "subs";
+ $self->free_ruleset_source($pms, 'head', $priority);
return unless ($evalstr);
@@ -609,6 +611,7 @@
# clear out a previous version of this fn
undef &{"${package_name}::_body_tests_${clean_priority}"};
+ $self->free_ruleset_source($pms, 'body', $priority);
return unless ($evalstr);
@@ -738,6 +741,7 @@
# clear out a previous version of this fn
undef &{"${package_name}::_body_uri_tests_${clean_priority}"};
+ $self->free_ruleset_source($pms, 'uri', $priority);
return unless ($evalstr);
@@ -868,6 +872,7 @@
# clear out a previous version of this fn
undef &{"${package_name}::_rawbody_tests_${clean_priority}"};
+ $self->free_ruleset_source($pms, 'rawbody', $priority);
return unless ($evalstr);
@@ -948,6 +953,7 @@
}
undef &{"${package_name}::_full_tests_${clean_priority}"};
+ $self->free_ruleset_source($pms, 'full', $priority);
return unless ($evalstr);
@@ -1143,6 +1149,9 @@
';
}
+ %{$evalhash} = ();
+ $self->free_ruleset_source($pms, 'eval', $priority);
+
# nothing done in the loop, that means no rules
return unless ($evalstr);
@@ -1268,6 +1277,19 @@
return '
$self->{main}->call_plugins ("ran_rule", { permsgstatus => $self, rulename => \''.$rulename.'\', ruletype => \''.$ruletype.'\' });
';
+}
+
+sub free_ruleset_source {
+ my ($self, $pms, $type, $pri) = @_;
+
+ # we can't do this, if we may need to recompile them again later
+ return if $pms->{conf}->{allow_user_rules};
+
+ # remove now-compiled rulesets
+ if (exists $pms->{conf}->{$type.'_tests'}->{$pri}) {
+ dbg "freeing ruleset source of type $type";
+ delete $pms->{conf}->{$type.'_tests'}->{$pri};
+ }
}
1;
Modified: spamassassin/trunk/masses/mass-check
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/mass-check?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/masses/mass-check (original)
+++ spamassassin/trunk/masses/mass-check Sun Nov 12 14:10:21 2006
@@ -741,13 +741,6 @@
undef $ma; # clean 'em up
undef $status;
- # uncomment these lines to get a Data::Dumper dump of the Mail::SpamAssassin
- # module, written to a file after each message is scanned. This is a
- # great way to find memory leaks...
- ## use Data::Dumper;
- ## open (D, ">dump.$$.$total_count"); print D Dumper($spamtest); close D;
- ## warn "wrote memory dump: dump.$$.$total_count";
-
showdots_blip();
# print ">>>> out = $out\n";
return $out;