You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by qu...@apache.org on 2004/03/19 08:46:33 UTC
svn commit: rev 9610 - in incubator/spamassassin/trunk: lib/Mail/SpamAssassin rules
Author: quinlan
Date: Thu Mar 18 23:46:31 2004
New Revision: 9610
Modified:
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
incubator/spamassassin/trunk/rules/20_body_tests.cf
incubator/spamassassin/trunk/rules/70_testing.cf
Log:
promote T_DOMAIN_RATIO_026 to DOMAIN_RATIO
at least cache URIs in get_uri_list()
tidy some code
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/EvalTests.pm Thu Mar 18 23:46:31 2004
@@ -3582,14 +3582,14 @@
###########################################################################
-sub domain_ratio {
+sub check_domain_ratio {
my($self, $body, $ratio) = @_;
my $length = (length(join('', @{$body})) || 1);
- if (!defined $self->{domains}) {
+ if (!defined $self->{uri_domain_count}) {
$self->get_uri_list();
}
- return 0 if !defined $self->{domains};
- return (($self->{domains} / $length) > $ratio);
+ return 0 if !defined $self->{uri_domain_count};
+ return (($self->{uri_domain_count} / $length) > $ratio);
}
###########################################################################
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Thu Mar 18 23:46:31 2004
@@ -1440,26 +1440,14 @@
# Finally, the address-spec regex (more or less)
my $Addr_spec_re = qr<$local_part\s*\@\s*$domain>o;
-# Discard all but one of identical successive entries in an array.
-# The input must be sorted if you want the returned array to be
-# without identical entries.
-sub _uniq {
- my $previous;
- my @uniq;
- if (@_) {
- push(@uniq, ($previous = shift(@_)));
- }
- foreach my $current (@_) {
- next if ($current eq $previous);
- push(@uniq, ($previous = $current));
- }
- return @uniq;
-}
-
+# This really belongs in metadata
sub get_uri_list {
my ($self, $textary) = @_;
- #$self->{found_bad_uri_encoding} = 0;
+ # use cached answer if available
+ if (defined $self->{uri_list}) {
+ return @{$self->{uri_list}};
+ }
$textary ||= $self->get_decoded_body_text_array();
my ($rulename, $pat, @uris);
@@ -1509,45 +1497,40 @@
push @uris, @{ $self->{html}{uri} };
}
- # Make sure we catch bad encoding tricks ...
- foreach my $uri ( @uris ) {
- next if ( $uri =~ /^mailto:/i );
+ # make sure we catch bad encoding tricks
+ for my $uri (@uris) {
+ next if $uri =~ /^mailto:/i;
- # bug 2844
# http://www.foo.biz?id=3 -> http://www.foo.biz/?id=3
$uri =~ s/^(https?:\/\/[^\/\?]+)\?/$1\/?/;
- # deal with encoding of chars ...
- # this is just the set of printable chars, minus ' ' (aka: dec 33-126, hex 21-7e)
- #
+ # deal with encoding of chars, this is just the set of printable
+ # chars minus ' ' (that is, dec 33-126, hex 21-7e)
$uri =~ s/\&\#0*(3[3-9]|[4-9]\d|1[01]\d|12[0-6]);/sprintf "%c",$1/e;
$uri =~ s/\&\#x0*(2[1-9]|[3-6][a-f0-9]|7[0-9a-e]);/sprintf "%c",hex($1)/ei;
- my($nuri, $unencoded, $encoded) = Mail::SpamAssassin::Util::URLEncode($uri);
- if ( $nuri ne $uri ) {
+ my ($nuri, $unencoded, $encoded) =
+ Mail::SpamAssassin::Util::url_encode($uri);
+ if ($nuri ne $uri) {
push(@uris, $nuri);
-
- # allow some unencodings to be ok ...
- # This is essentially HTTP_EXCESSIVE_ESCAPES ...
- #if ( $unencoded =~ /[a-zA-Z0-9\/]/ ) {
- # $self->{found_bad_uri_encoding} = 1;
- #}
}
}
# remove duplicates
- @uris = _uniq(sort(@uris));
+ my %uris;
+ $uris{$_} = 1 for @uris;
+ @uris = keys %uris;
# get domain list
- my @domains;
+ my %domains;
for (@uris) {
my $domain = Mail::SpamAssassin::Util::uri_to_domain($_);
- push @domains, $domain if $domain;
+ $domains{$domain} = 1 if $domain;
}
- @domains = _uniq(sort(@domains));
- $self->{domains} = scalar @domains;
+ $self->{uri_domain_count} = keys %domains;
$self->{uri_list} = \@uris;
+
dbg("uri tests: Done uriRE");
return @{$self->{uri_list}};
}
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Util.pm Thu Mar 18 23:46:31 2004
@@ -601,37 +601,39 @@
###########################################################################
-sub URLEncode {
- my($url)=@_;
- my(@characters)=split(/(\%[0-9a-fA-F]{2})/,$url);
- my(@unencoded) = ();
- my(@encoded) = ();
+sub url_encode {
+ my ($url) = @_;
+ my (@characters) = split(/(\%[0-9a-fA-F]{2})/, $url);
+ my (@unencoded) = ();
+ my (@encoded) = ();
- foreach(@characters) {
- if ( /\%[0-9a-fA-F]{2}/ ) { # Escaped character set ...
- # IF it is in the range of 0x00-0x20 or 0x7f-0xff
- # or it is one of "<", ">", """, "#", "%",
- # ";", "/", "?", ":", "@", "=" or "&"
- # THEN preserve its encoding
- unless ( /(20|7f|[0189a-fA-F][0-9a-fA-F])/i
- || /2[2356fF]|3[a-fA-F]|40/i )
- {
- s/\%([2-7][0-9a-fA-F])/sprintf "%c",hex($1)/e;
- push(@unencoded, $_);
- }
- }
- else { # Other stuff
- # 0x00-0x20, 0x7f-0xff, <, >, and " ... "
- s/([\000-\040\177-\377\074\076\042])
- /push(@encoded,$1) && sprintf "%%%02x",unpack("C",$1)/egx;
- }
- }
- if (wantarray) {
- return(join("",@characters), join("",@unencoded), join("",@encoded));
+ foreach (@characters) {
+ # escaped character set ...
+ if (/\%[0-9a-fA-F]{2}/) {
+ # IF it is in the range of 0x00-0x20 or 0x7f-0xff
+ # or it is one of "<", ">", """, "#", "%",
+ # ";", "/", "?", ":", "@", "=" or "&"
+ # THEN preserve its encoding
+ unless (/(20|7f|[0189a-fA-F][0-9a-fA-F])/i
+ || /2[2356fF]|3[a-fA-F]|40/i)
+ {
+ s/\%([2-7][0-9a-fA-F])/sprintf "%c", hex($1)/e;
+ push(@unencoded, $_);
+ }
}
+ # other stuff
else {
- return join("",@characters);
+ # 0x00-0x20, 0x7f-0xff, <, >, and " ... "
+ s/([\000-\040\177-\377\074\076\042])
+ /push(@encoded, $1) && sprintf "%%%02x", unpack("C",$1)/egx;
}
+ }
+ if (wantarray) {
+ return(join("", @characters), join("", @unencoded), join("", @encoded));
+ }
+ else {
+ return join("", @characters);
+ }
}
###########################################################################
Modified: incubator/spamassassin/trunk/rules/20_body_tests.cf
==============================================================================
--- incubator/spamassassin/trunk/rules/20_body_tests.cf (original)
+++ incubator/spamassassin/trunk/rules/20_body_tests.cf Thu Mar 18 23:46:31 2004
@@ -197,3 +197,6 @@
body UNIQUE_WORDS eval:check_unique_words('0.946', '3.1')
describe UNIQUE_WORDS Message body has many words used only once
+
+body DOMAIN_RATIO eval:check_domain_ratio('0.026')
+describe DOMAIN_RATIO Message body mentions many internet domains
Modified: incubator/spamassassin/trunk/rules/70_testing.cf
==============================================================================
--- incubator/spamassassin/trunk/rules/70_testing.cf (original)
+++ incubator/spamassassin/trunk/rules/70_testing.cf Thu Mar 18 23:46:31 2004
@@ -368,57 +368,6 @@
uri T_LONG_HOSTNAME_30 /^https?:\/\/[a-z0-9]{30,40}\./i
uri T_LONG_HOSTNAME_40 /^https?:\/\/[a-z0-9]{40,}\./i
-body T_DOMAIN_RATIO_001 eval:domain_ratio('0.001')
-body T_DOMAIN_RATIO_002 eval:domain_ratio('0.002')
-body T_DOMAIN_RATIO_003 eval:domain_ratio('0.003')
-body T_DOMAIN_RATIO_004 eval:domain_ratio('0.004')
-body T_DOMAIN_RATIO_005 eval:domain_ratio('0.005')
-body T_DOMAIN_RATIO_006 eval:domain_ratio('0.006')
-body T_DOMAIN_RATIO_007 eval:domain_ratio('0.007')
-body T_DOMAIN_RATIO_008 eval:domain_ratio('0.008')
-body T_DOMAIN_RATIO_009 eval:domain_ratio('0.009')
-body T_DOMAIN_RATIO_010 eval:domain_ratio('0.010')
-body T_DOMAIN_RATIO_011 eval:domain_ratio('0.011')
-body T_DOMAIN_RATIO_012 eval:domain_ratio('0.012')
-body T_DOMAIN_RATIO_013 eval:domain_ratio('0.013')
-body T_DOMAIN_RATIO_014 eval:domain_ratio('0.014')
-body T_DOMAIN_RATIO_015 eval:domain_ratio('0.015')
-body T_DOMAIN_RATIO_016 eval:domain_ratio('0.016')
-body T_DOMAIN_RATIO_017 eval:domain_ratio('0.017')
-body T_DOMAIN_RATIO_018 eval:domain_ratio('0.018')
-body T_DOMAIN_RATIO_019 eval:domain_ratio('0.019')
-body T_DOMAIN_RATIO_020 eval:domain_ratio('0.020')
-body T_DOMAIN_RATIO_021 eval:domain_ratio('0.021')
-body T_DOMAIN_RATIO_022 eval:domain_ratio('0.022')
-body T_DOMAIN_RATIO_023 eval:domain_ratio('0.023')
-body T_DOMAIN_RATIO_024 eval:domain_ratio('0.024')
-body T_DOMAIN_RATIO_025 eval:domain_ratio('0.025')
-body T_DOMAIN_RATIO_026 eval:domain_ratio('0.026')
-body T_DOMAIN_RATIO_027 eval:domain_ratio('0.027')
-body T_DOMAIN_RATIO_028 eval:domain_ratio('0.028')
-body T_DOMAIN_RATIO_029 eval:domain_ratio('0.029')
-body T_DOMAIN_RATIO_030 eval:domain_ratio('0.030')
-body T_DOMAIN_RATIO_031 eval:domain_ratio('0.031')
-body T_DOMAIN_RATIO_032 eval:domain_ratio('0.032')
-body T_DOMAIN_RATIO_033 eval:domain_ratio('0.033')
-body T_DOMAIN_RATIO_034 eval:domain_ratio('0.034')
-body T_DOMAIN_RATIO_035 eval:domain_ratio('0.035')
-body T_DOMAIN_RATIO_036 eval:domain_ratio('0.036')
-body T_DOMAIN_RATIO_037 eval:domain_ratio('0.037')
-body T_DOMAIN_RATIO_038 eval:domain_ratio('0.038')
-body T_DOMAIN_RATIO_039 eval:domain_ratio('0.039')
-body T_DOMAIN_RATIO_040 eval:domain_ratio('0.040')
-body T_DOMAIN_RATIO_041 eval:domain_ratio('0.041')
-body T_DOMAIN_RATIO_042 eval:domain_ratio('0.042')
-body T_DOMAIN_RATIO_043 eval:domain_ratio('0.043')
-body T_DOMAIN_RATIO_044 eval:domain_ratio('0.044')
-body T_DOMAIN_RATIO_045 eval:domain_ratio('0.045')
-body T_DOMAIN_RATIO_046 eval:domain_ratio('0.046')
-body T_DOMAIN_RATIO_047 eval:domain_ratio('0.047')
-body T_DOMAIN_RATIO_048 eval:domain_ratio('0.048')
-body T_DOMAIN_RATIO_049 eval:domain_ratio('0.049')
-body T_DOMAIN_RATIO_050 eval:domain_ratio('0.050')
-
# bug 2220
meta T_DIGEST_MULTIPLE RAZOR2_CHECK + DCC_CHECK + PYZOR_CHECK > 1
describe T_DIGEST_MULTIPLE Message hits more than one network digest check