You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2006/11/12 23:10:22 UTC

svn commit: r474059 - in /spamassassin/trunk: lib/Mail/SpamAssassin.pm lib/Mail/SpamAssassin/Conf.pm lib/Mail/SpamAssassin/Conf/Parser.pm lib/Mail/SpamAssassin/PerMsgStatus.pm lib/Mail/SpamAssassin/Plugin/Check.pm masses/mass-check

Author: jm
Date: Sun Nov 12 14:10:21 2006
New Revision: 474059

URL: http://svn.apache.org/viewvc?view=rev&rev=474059
Log:
reduce memory footprint by about 750KB by: deleting the source for compiled rulesets; deleting stuff used to parse config; compacting the descriptions hash into a single string, for more RAM-efficient but slower lookups

Modified:
    spamassassin/trunk/lib/Mail/SpamAssassin.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
    spamassassin/trunk/masses/mass-check

Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin.pm Sun Nov 12 14:10:21 2006
@@ -430,7 +430,6 @@
 
 sub check {
   my ($self, $mail_obj) = @_;
-  local ($_);
 
   $self->init(1);
   my $msg = Mail::SpamAssassin::PerMsgStatus->new($self, $mail_obj);
@@ -1227,6 +1226,8 @@
   $mail->finish();
   $self->finish_learner();
 
+  $self->{conf}->free_uncompiled_rule_source();
+
   # load SQL modules now as well
   my $dsn = $self->{conf}->{user_scores_dsn};
   if ($dsn ne '') {
@@ -1399,7 +1400,7 @@
     }
 
     if ($sysrules) {
-      $self->{config_text} .= $self->read_cf ($sysrules, 'default rules dir');
+      $self->{config_text} .= $self->read_cf($sysrules, 'default rules dir');
     }
 
     if (!$self->{languages_filename}) {
@@ -1407,7 +1408,7 @@
     }
 
     if ($siterules) {
-      $self->{config_text} .= $self->read_cf ($siterules, 'site rules dir');
+      $self->{config_text} .= $self->read_cf($siterules, 'site rules dir');
     }
 
     if ( $use_user_pref != 0 ) {
@@ -1427,7 +1428,7 @@
         }
       }
 
-      $self->{config_text} .= $self->read_cf ($fname, 'user prefs file');
+      $self->{config_text} .= $self->read_cf($fname, 'user prefs file');
     }
   }
 

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Sun Nov 12 14:10:21 2006
@@ -2893,6 +2893,17 @@
 
 ###########################################################################
 
+sub get_description_for_rule {
+  my ($self, $rule) = @_;
+  if ($self->{descriptions_str} =~ /^\Q${rule}\E:(.*?)$/m) {
+    return $1;
+  } else {
+    return;
+  }
+}
+
+###########################################################################
+
 sub maybe_header_only {
   my($self,$rulename) = @_;
   my $type = $self->{test_types}->{$rulename};
@@ -3031,6 +3042,15 @@
   delete $dest->{sed_path_cache};
 
   return 1;
+}
+
+###########################################################################
+
+sub free_uncompiled_rule_source {
+  my ($self) = @_;
+  delete $self->{if_stack};
+  delete $self->{source_file};
+  delete $self->{meta_dependencies};
 }
 
 ###########################################################################

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf/Parser.pm Sun Nov 12 14:10:21 2006
@@ -663,6 +663,7 @@
 
   $self->trace_meta_dependencies();
   $self->fix_priorities();
+  $self->compact_descriptions();
 
   dbg("conf: finish parsing");
 
@@ -750,8 +751,11 @@
   # named this way just in case we ever want a "finish_parsing_start"
   $conf->{main}->call_plugins("finish_parsing_end", { conf => $conf });
 
-  delete $conf->{tests};                # free it up
-  delete $conf->{priority};             # free it up
+  # free up stuff we no longer need
+  delete $conf->{tests};
+  delete $conf->{priority};
+  delete $conf->{test_types};
+  # delete $conf->{tflags};
 }
 
 sub trace_meta_dependencies {
@@ -831,6 +835,24 @@
       }
     }
   }
+}
+
+# compact the {descriptions} hash into a single string; by using
+# a string, quite a lot of RAM is freed up in exchange for slightly
+# slower lookup time
+sub compact_descriptions {
+  my ($self) = @_;
+  my $conf = $self->{conf};
+
+  my $descs = '';
+  my ($k, $v);
+  while (($k, $v) = each %{$conf->{descriptions}})
+  {
+    $descs .= "$k:$v\n";
+  }
+
+  delete $conf->{descriptions};
+  $conf->{descriptions_str} = $descs;
 }
 
 ###########################################################################

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Sun Nov 12 14:10:21 2006
@@ -2114,7 +2114,7 @@
             $params{score} || $self->{conf}->{scores}->{$rule},
             $area,
             $params{ruletype},
-            ($self->{conf}->{descriptions}->{$rule} || $rule));
+            $self->{conf}->get_description_for_rule($rule) || $rule);
 }
 
 ###########################################################################

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Sun Nov 12 14:10:21 2006
@@ -318,6 +318,7 @@
   no strict "subs";
   undef &{"${package_name}::_meta_tests_${clean_priority}"};
   use strict "subs";
+  $self->free_ruleset_source($pms, 'meta', $priority);
 
   return unless ($evalstr);
 
@@ -473,6 +474,7 @@
   no strict "subs";
   undef &{"${package_name}::_head_tests_${clean_priority}"};
   use strict "subs";
+  $self->free_ruleset_source($pms, 'head', $priority);
 
   return unless ($evalstr);
 
@@ -609,6 +611,7 @@
 
   # clear out a previous version of this fn
   undef &{"${package_name}::_body_tests_${clean_priority}"};
+  $self->free_ruleset_source($pms, 'body', $priority);
 
   return unless ($evalstr);
 
@@ -738,6 +741,7 @@
 
   # clear out a previous version of this fn
   undef &{"${package_name}::_body_uri_tests_${clean_priority}"};
+  $self->free_ruleset_source($pms, 'uri', $priority);
 
   return unless ($evalstr);
 
@@ -868,6 +872,7 @@
 
   # clear out a previous version of this fn
   undef &{"${package_name}::_rawbody_tests_${clean_priority}"};
+  $self->free_ruleset_source($pms, 'rawbody', $priority);
 
   return unless ($evalstr);
 
@@ -948,6 +953,7 @@
   }
 
   undef &{"${package_name}::_full_tests_${clean_priority}"};
+  $self->free_ruleset_source($pms, 'full', $priority);
 
   return unless ($evalstr);
 
@@ -1143,6 +1149,9 @@
     ';
   }
 
+  %{$evalhash} = ();
+  $self->free_ruleset_source($pms, 'eval', $priority);
+
   # nothing done in the loop, that means no rules 
   return unless ($evalstr);
  
@@ -1268,6 +1277,19 @@
   return '
     $self->{main}->call_plugins ("ran_rule", { permsgstatus => $self, rulename => \''.$rulename.'\', ruletype => \''.$ruletype.'\' });
   ';
+}
+
+sub free_ruleset_source {
+  my ($self, $pms, $type, $pri) = @_;
+
+  # we can't do this, if we may need to recompile them again later
+  return if $pms->{conf}->{allow_user_rules};
+
+  # remove now-compiled rulesets
+  if (exists $pms->{conf}->{$type.'_tests'}->{$pri}) {
+    dbg "freeing ruleset source of type $type";
+    delete $pms->{conf}->{$type.'_tests'}->{$pri};
+  }
 }
 
 1;

Modified: spamassassin/trunk/masses/mass-check
URL: http://svn.apache.org/viewvc/spamassassin/trunk/masses/mass-check?view=diff&rev=474059&r1=474058&r2=474059
==============================================================================
--- spamassassin/trunk/masses/mass-check (original)
+++ spamassassin/trunk/masses/mass-check Sun Nov 12 14:10:21 2006
@@ -741,13 +741,6 @@
   undef $ma;		# clean 'em up
   undef $status;
 
-  # uncomment these lines to get a Data::Dumper dump of the Mail::SpamAssassin
-  # module, written to a file after each message is scanned.  This is a
-  # great way to find memory leaks...
-  ## use Data::Dumper;
-  ## open (D, ">dump.$$.$total_count"); print D Dumper($spamtest); close D;
-  ## warn "wrote memory dump: dump.$$.$total_count";
-
   showdots_blip();
 #  print ">>>> out = $out\n";
   return $out;