You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by qu...@apache.org on 2004/03/19 08:46:33 UTC

svn commit: rev 9610 - in incubator/spamassassin/trunk: lib/Mail/SpamAssassin rules

Author: quinlan
Date: Thu Mar 18 23:46:31 2004
New Revision: 9610

Modified:
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
   incubator/spamassassin/trunk/rules/20_body_tests.cf
   incubator/spamassassin/trunk/rules/70_testing.cf
Log:
promote T_DOMAIN_RATIO_026 to DOMAIN_RATIO
at least cache URIs in get_uri_list()
tidy some code


Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm	Thu Mar 18 23:46:31 2004
@@ -3582,14 +3582,14 @@
 
 ###########################################################################
 
-sub domain_ratio {
+sub check_domain_ratio {
   my($self, $body, $ratio) = @_;
   my $length = (length(join('', @{$body})) || 1);
-  if (!defined $self->{domains}) {
+  if (!defined $self->{uri_domain_count}) {
     $self->get_uri_list();
   }
-  return 0 if !defined $self->{domains};
-  return (($self->{domains} / $length) > $ratio);
+  return 0 if !defined $self->{uri_domain_count};
+  return (($self->{uri_domain_count} / $length) > $ratio);
 }
 
 ###########################################################################

Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	Thu Mar 18 23:46:31 2004
@@ -1440,26 +1440,14 @@
 # Finally, the address-spec regex (more or less)
 my $Addr_spec_re   = qr<$local_part\s*\@\s*$domain>o;
 
-# Discard all but one of identical successive entries in an array.
-# The input must be sorted if you want the returned array to be
-# without identical entries.
-sub _uniq {
-  my $previous;
-  my @uniq;
-  if (@_) {
-    push(@uniq, ($previous = shift(@_)));
-  }
-  foreach my $current (@_) {
-    next if ($current eq $previous);
-    push(@uniq, ($previous = $current));
-  }
-  return @uniq;
-}
-
+# This really belongs in metadata
 sub get_uri_list {
   my ($self, $textary) = @_;
 
-  #$self->{found_bad_uri_encoding} = 0;
+  # use cached answer if available
+  if (defined $self->{uri_list}) {
+    return @{$self->{uri_list}};
+  }
 
   $textary ||= $self->get_decoded_body_text_array();
   my ($rulename, $pat, @uris);
@@ -1509,45 +1497,40 @@
     push @uris, @{ $self->{html}{uri} };
   }
 
-  # Make sure we catch bad encoding tricks ...
-  foreach my $uri ( @uris ) {
-    next if ( $uri =~ /^mailto:/i );
+  # make sure we catch bad encoding tricks
+  for my $uri (@uris) {
+    next if $uri =~ /^mailto:/i;
 
-    # bug 2844
     # http://www.foo.biz?id=3 -> http://www.foo.biz/?id=3
     $uri =~ s/^(https?:\/\/[^\/\?]+)\?/$1\/?/;
 
-    # deal with encoding of chars ...
-    # this is just the set of printable chars, minus ' ' (aka: dec 33-126, hex 21-7e)
-    #
+    # deal with encoding of chars, this is just the set of printable
+    # chars minus ' ' (that is, dec 33-126, hex 21-7e)
     $uri =~ s/\&\#0*(3[3-9]|[4-9]\d|1[01]\d|12[0-6]);/sprintf "%c",$1/e;
     $uri =~ s/\&\#x0*(2[1-9]|[3-6][a-f0-9]|7[0-9a-e]);/sprintf "%c",hex($1)/ei;
 
-    my($nuri, $unencoded, $encoded) = Mail::SpamAssassin::Util::URLEncode($uri);
-    if ( $nuri ne $uri ) {
+    my ($nuri, $unencoded, $encoded) =
+      Mail::SpamAssassin::Util::url_encode($uri);
+    if ($nuri ne $uri) {
       push(@uris, $nuri);
-
-      # allow some unencodings to be ok ...
-      # This is essentially HTTP_EXCESSIVE_ESCAPES ...
-      #if ( $unencoded =~ /[a-zA-Z0-9\/]/ ) {
-      #  $self->{found_bad_uri_encoding} = 1;
-      #}
     }
   }
 
   # remove duplicates
-  @uris = _uniq(sort(@uris));
+  my %uris;
+  $uris{$_} = 1 for @uris;
+  @uris = keys %uris;
 
   # get domain list
-  my @domains;
+  my %domains;
   for (@uris) {
     my $domain = Mail::SpamAssassin::Util::uri_to_domain($_);
-    push @domains, $domain if $domain;
+    $domains{$domain} = 1 if $domain;
   }
-  @domains = _uniq(sort(@domains));
-  $self->{domains} = scalar @domains;
 
+  $self->{uri_domain_count} = keys %domains;
   $self->{uri_list} = \@uris;
+
   dbg("uri tests: Done uriRE");
   return @{$self->{uri_list}};
 }

Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm	Thu Mar 18 23:46:31 2004
@@ -601,37 +601,39 @@
 
 ###########################################################################
 
-sub URLEncode {
-    my($url)=@_;
-    my(@characters)=split(/(\%[0-9a-fA-F]{2})/,$url);
-    my(@unencoded) = ();
-    my(@encoded) = ();
+sub url_encode {
+  my ($url) = @_;
+  my (@characters) = split(/(\%[0-9a-fA-F]{2})/, $url);
+  my (@unencoded) = ();
+  my (@encoded) = ();
 
-    foreach(@characters) {
-	if ( /\%[0-9a-fA-F]{2}/ ) {		# Escaped character set ...
-	    # IF it is in the range of 0x00-0x20 or 0x7f-0xff
-	    #    or it is one of  "<", ">", """, "#", "%",
-	    #                     ";", "/", "?", ":", "@", "=" or "&"
-	    # THEN preserve its encoding
-	    unless ( /(20|7f|[0189a-fA-F][0-9a-fA-F])/i
-		    || /2[2356fF]|3[a-fA-F]|40/i )
-	    {
-		s/\%([2-7][0-9a-fA-F])/sprintf "%c",hex($1)/e;
-		push(@unencoded, $_);
-	    }
-	}
-	else {					# Other stuff
-	    # 0x00-0x20, 0x7f-0xff, <, >, and " ... "
-	    s/([\000-\040\177-\377\074\076\042])
-	     /push(@encoded,$1) && sprintf "%%%02x",unpack("C",$1)/egx;
-	}
-    }
-    if (wantarray) {
-      return(join("",@characters), join("",@unencoded), join("",@encoded));
+  foreach (@characters) {
+    # escaped character set ...
+    if (/\%[0-9a-fA-F]{2}/) {
+      # IF it is in the range of 0x00-0x20 or 0x7f-0xff
+      #    or it is one of  "<", ">", """, "#", "%",
+      #                     ";", "/", "?", ":", "@", "=" or "&"
+      # THEN preserve its encoding
+      unless (/(20|7f|[0189a-fA-F][0-9a-fA-F])/i
+	      || /2[2356fF]|3[a-fA-F]|40/i)
+      {
+	s/\%([2-7][0-9a-fA-F])/sprintf "%c", hex($1)/e;
+	push(@unencoded, $_);
+      }
     }
+    # other stuff
     else {
-      return join("",@characters);
+      # 0x00-0x20, 0x7f-0xff, <, >, and " ... "
+      s/([\000-\040\177-\377\074\076\042])
+	  /push(@encoded, $1) && sprintf "%%%02x", unpack("C",$1)/egx;
     }
+  }
+  if (wantarray) {
+    return(join("", @characters), join("", @unencoded), join("", @encoded));
+  }
+  else {
+    return join("", @characters);
+  }
 }
 
 ###########################################################################

Modified: incubator/spamassassin/trunk/rules/20_body_tests.cf
==============================================================================
--- incubator/spamassassin/trunk/rules/20_body_tests.cf	(original)
+++ incubator/spamassassin/trunk/rules/20_body_tests.cf	Thu Mar 18 23:46:31 2004
@@ -197,3 +197,6 @@
 
 body UNIQUE_WORDS	eval:check_unique_words('0.946', '3.1')
 describe UNIQUE_WORDS	Message body has many words used only once
+
+body DOMAIN_RATIO	eval:check_domain_ratio('0.026')
+describe DOMAIN_RATIO	Message body mentions many internet domains

Modified: incubator/spamassassin/trunk/rules/70_testing.cf
==============================================================================
--- incubator/spamassassin/trunk/rules/70_testing.cf	(original)
+++ incubator/spamassassin/trunk/rules/70_testing.cf	Thu Mar 18 23:46:31 2004
@@ -368,57 +368,6 @@
 uri T_LONG_HOSTNAME_30		/^https?:\/\/[a-z0-9]{30,40}\./i
 uri T_LONG_HOSTNAME_40		/^https?:\/\/[a-z0-9]{40,}\./i
 
-body T_DOMAIN_RATIO_001		eval:domain_ratio('0.001')
-body T_DOMAIN_RATIO_002		eval:domain_ratio('0.002')
-body T_DOMAIN_RATIO_003		eval:domain_ratio('0.003')
-body T_DOMAIN_RATIO_004		eval:domain_ratio('0.004')
-body T_DOMAIN_RATIO_005		eval:domain_ratio('0.005')
-body T_DOMAIN_RATIO_006		eval:domain_ratio('0.006')
-body T_DOMAIN_RATIO_007		eval:domain_ratio('0.007')
-body T_DOMAIN_RATIO_008		eval:domain_ratio('0.008')
-body T_DOMAIN_RATIO_009		eval:domain_ratio('0.009')
-body T_DOMAIN_RATIO_010		eval:domain_ratio('0.010')
-body T_DOMAIN_RATIO_011		eval:domain_ratio('0.011')
-body T_DOMAIN_RATIO_012		eval:domain_ratio('0.012')
-body T_DOMAIN_RATIO_013		eval:domain_ratio('0.013')
-body T_DOMAIN_RATIO_014		eval:domain_ratio('0.014')
-body T_DOMAIN_RATIO_015		eval:domain_ratio('0.015')
-body T_DOMAIN_RATIO_016		eval:domain_ratio('0.016')
-body T_DOMAIN_RATIO_017		eval:domain_ratio('0.017')
-body T_DOMAIN_RATIO_018		eval:domain_ratio('0.018')
-body T_DOMAIN_RATIO_019		eval:domain_ratio('0.019')
-body T_DOMAIN_RATIO_020		eval:domain_ratio('0.020')
-body T_DOMAIN_RATIO_021		eval:domain_ratio('0.021')
-body T_DOMAIN_RATIO_022		eval:domain_ratio('0.022')
-body T_DOMAIN_RATIO_023		eval:domain_ratio('0.023')
-body T_DOMAIN_RATIO_024		eval:domain_ratio('0.024')
-body T_DOMAIN_RATIO_025		eval:domain_ratio('0.025')
-body T_DOMAIN_RATIO_026		eval:domain_ratio('0.026')
-body T_DOMAIN_RATIO_027		eval:domain_ratio('0.027')
-body T_DOMAIN_RATIO_028		eval:domain_ratio('0.028')
-body T_DOMAIN_RATIO_029		eval:domain_ratio('0.029')
-body T_DOMAIN_RATIO_030		eval:domain_ratio('0.030')
-body T_DOMAIN_RATIO_031		eval:domain_ratio('0.031')
-body T_DOMAIN_RATIO_032		eval:domain_ratio('0.032')
-body T_DOMAIN_RATIO_033		eval:domain_ratio('0.033')
-body T_DOMAIN_RATIO_034		eval:domain_ratio('0.034')
-body T_DOMAIN_RATIO_035		eval:domain_ratio('0.035')
-body T_DOMAIN_RATIO_036		eval:domain_ratio('0.036')
-body T_DOMAIN_RATIO_037		eval:domain_ratio('0.037')
-body T_DOMAIN_RATIO_038		eval:domain_ratio('0.038')
-body T_DOMAIN_RATIO_039		eval:domain_ratio('0.039')
-body T_DOMAIN_RATIO_040		eval:domain_ratio('0.040')
-body T_DOMAIN_RATIO_041		eval:domain_ratio('0.041')
-body T_DOMAIN_RATIO_042		eval:domain_ratio('0.042')
-body T_DOMAIN_RATIO_043		eval:domain_ratio('0.043')
-body T_DOMAIN_RATIO_044		eval:domain_ratio('0.044')
-body T_DOMAIN_RATIO_045		eval:domain_ratio('0.045')
-body T_DOMAIN_RATIO_046		eval:domain_ratio('0.046')
-body T_DOMAIN_RATIO_047		eval:domain_ratio('0.047')
-body T_DOMAIN_RATIO_048		eval:domain_ratio('0.048')
-body T_DOMAIN_RATIO_049		eval:domain_ratio('0.049')
-body T_DOMAIN_RATIO_050		eval:domain_ratio('0.050')
-
 # bug 2220
 meta T_DIGEST_MULTIPLE		RAZOR2_CHECK + DCC_CHECK + PYZOR_CHECK > 1
 describe T_DIGEST_MULTIPLE	Message hits more than one network digest check