You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by he...@apache.org on 2018/09/26 12:07:30 UTC
svn commit: r1842015 - in /spamassassin/trunk: MANIFEST
lib/Mail/SpamAssassin/Plugin/URILocalBL.pm t/data/spam/urilocalbl_net.eml
t/urilocalbl.t t/urilocalbl_dbfile.t t/urilocalbl_fast.t
t/urilocalbl_geoip.t t/urilocalbl_geoip2.t
Author: hege
Date: Wed Sep 26 12:07:29 2018
New Revision: 1842015
URL: http://svn.apache.org/viewvc?rev=1842015&view=rev
Log:
Make URILocalBL use internal async DNS, make IPv6 lookups, combine tests to single file
Added:
spamassassin/trunk/t/data/spam/urilocalbl_net.eml
spamassassin/trunk/t/urilocalbl.t
Removed:
spamassassin/trunk/t/urilocalbl_dbfile.t
spamassassin/trunk/t/urilocalbl_fast.t
spamassassin/trunk/t/urilocalbl_geoip.t
spamassassin/trunk/t/urilocalbl_geoip2.t
Modified:
spamassassin/trunk/MANIFEST
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm
Modified: spamassassin/trunk/MANIFEST
URL: http://svn.apache.org/viewvc/spamassassin/trunk/MANIFEST?rev=1842015&r1=1842014&r2=1842015&view=diff
==============================================================================
--- spamassassin/trunk/MANIFEST (original)
+++ spamassassin/trunk/MANIFEST Wed Sep 26 12:07:29 2018
@@ -558,10 +558,7 @@ t/uri_html.t
t/uri_list.t
t/uri_text.t
t/uribl.t
-t/urilocalbl_dbfile.t
-t/urilocalbl_fast.t
-t/urilocalbl_geoip.t
-t/urilocalbl_geoip2.t
+t/urilocalbl.t
t/utf8.t
t/util_wrap.t
t/whitelist_addrs.t
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm?rev=1842015&r1=1842014&r2=1842015&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm Wed Sep 26 12:07:29 2018
@@ -122,7 +122,10 @@ use version;
our @ISA = qw(Mail::SpamAssassin::Plugin);
-sub dbg { Mail::SpamAssassin::Plugin::dbg ("URILocalBL: @_"); }
+sub dbg {
+ my $str = shift;
+ Mail::SpamAssassin::Plugin::dbg ("URILocalBL: $str", @_);
+}
my $IP_ADDRESS = IP_ADDRESS;
@@ -331,11 +334,8 @@ sub finish_parsing_end {
}
}
-sub check_uri_local_bl {
+sub _init_geodb {
my ($self, $pms) = @_;
-
- return 0 if $self->{urilocalbl_disabled};
-
if (!$self->{geodb}) {
eval {
$self->{geodb} = Mail::SpamAssassin::GeoDB->new({
@@ -349,14 +349,21 @@ sub check_uri_local_bl {
return 0;
}
}
- my $geodb = $self->{geodb};
+ return 1;
+}
+
+sub check_uri_local_bl {
+ my ($self, $pms) = @_;
+
+ return 0 if $self->{urilocalbl_disabled};
+ return 0 if !$self->_init_geodb($pms);
my $rulename = $pms->get_current_eval_rule_name();
my $ruleconf = $pms->{conf}->{urilocalbl}->{$rulename};
dbg("running $rulename");
- my @addrs;
+ my %found_hosts;
foreach my $info (values %{$pms->get_uri_detail_list()}) {
next unless $info->{hosts};
@@ -370,105 +377,160 @@ sub check_uri_local_bl {
dbg("excluded $host, domain $domain matches");
next HOST;
}
-
- if ($host !~ /^$IP_ADDRESS$/o) {
- # this would be best cached from prior lookups
- # TODO async extract_metadata lookup
- @addrs = gethostbyname($host);
- # convert to string values address list
- @addrs = map { inet_ntoa($_); } @addrs[4..$#addrs];
- if (@addrs) {
- dbg("$host IP-addresses: ".join(', ', @addrs));
- } else {
- dbg("$host failed to resolve IP-addresses");
+ if ($host =~ /^$IP_ADDRESS$/o) {
+ if ($self->_check_host($pms, $rulename, $host, [$host])) {
+ # if hit, rule is done
+ return 0;
}
} else {
- @addrs = ($host);
+ # do host lookups only after all IPs are checked, since they
+ # don't need resolving..
+ $found_hosts{$host} = 1;
}
+ }
+ }
- next HOST unless @addrs;
+ foreach my $host (keys %found_hosts) {
+ # launch dns
+ $pms->{async}->bgsend_and_start_lookup("$host.", 'A', undef,
+ { key => "urilocalbl:$host:A", host => $host, rulename => $rulename },
+ sub { my($ent, $pkt) = @_;
+ $self->_finish_lookup($pms, $ent, $pkt); },
+ master_deadline => $pms->{master_deadline}
+ );
+ # also IPv6 if database supports
+ $pms->{async}->bgsend_and_start_lookup("$host.", 'AAAA', undef,
+ { key => "urilocalbl:$host:AAAA", host => $host, rulename => $rulename },
+ sub { my($ent, $pkt) = @_;
+ $self->_finish_lookup($pms, $ent, $pkt); },
+ master_deadline => $pms->{master_deadline}
+ ) if $self->{geodb}->can('country_v6');
+ }
+}
- foreach my $ip (@addrs) {
- if (defined $ruleconf->{exclusions}{$ip}) {
- dbg("excluded $host, IP $ip matches");
- next HOST;
- }
- }
+sub _finish_lookup {
+ my ($self, $pms, $ent, $pkt) = @_;
- if (defined $ruleconf->{countries}) {
- my $neg = defined $ruleconf->{countries_neg};
- my $testcc = join(' ', sort keys %{$ruleconf->{countries}});
- if ($neg) {
- dbg("checking $host for any country except: $testcc");
- } else {
- dbg("checking $host for countries: $testcc");
- }
- foreach my $ip (@addrs) {
- my $cc = $geodb->get_country($ip);
- if ( (!$neg && defined $ruleconf->{countries}{$cc}) ||
- ($neg && !defined $ruleconf->{countries}{$cc}) ) {
- dbg("$host ($ip) country $cc - HIT");
- $pms->test_log("Host: $host in country $cc");
- return 1; # hit
- } else {
- dbg("$host ($ip) country $cc - ".($neg ? "excluded" : "no match"));
- }
- }
+ my $rulename = $ent->{rulename};
+ my $host = $ent->{host};
+
+ # Skip duplicate A / AAAA matches
+ return if $pms->{urilocalbl_finished}->{$rulename};
+
+ if (!$pkt) {
+ # $pkt will be undef if the DNS query was aborted (e.g. timed out)
+ dbg("host lookup failed: $rulename $host");
+ return;
+ }
+
+ my @answer = $pkt->answer;
+ my @addrs;
+ foreach my $rr (@answer) {
+ if ($rr->type eq 'A' || $rr->type eq 'AAAA') {
+ push @addrs, $rr->address;
+ }
+ }
+
+ if (@addrs) {
+ if ($self->_check_host($pms, $rulename, $host, \@addrs)) {
+ $pms->register_async_rule_finish($rulename);
+ $pms->{urilocalbl_finished}->{$rulename} = 1;
+ }
+ }
+}
+
+sub _check_host {
+ my ($self, $pms, $rulename, $host, $addrs) = @_;
+
+ my $ruleconf = $pms->{conf}->{urilocalbl}->{$rulename};
+
+ if ($host ne $addrs->[0]) {
+ dbg("resolved $host: ".join(', ', @$addrs));
+ }
+
+ foreach my $ip (@$addrs) {
+ if (defined $ruleconf->{exclusions}{$ip}) {
+ dbg("excluded $host, IP $ip matches");
+ return 1;
+ }
+ }
+
+ if (defined $ruleconf->{countries}) {
+ my $neg = defined $ruleconf->{countries_neg};
+ my $testcc = join(' ', sort keys %{$ruleconf->{countries}});
+ if ($neg) {
+ dbg("checking $host for any country except: $testcc");
+ } else {
+ dbg("checking $host for countries: $testcc");
+ }
+ foreach my $ip (@$addrs) {
+ my $cc = $self->{geodb}->get_country($ip);
+ if ( (!$neg && defined $ruleconf->{countries}{$cc}) ||
+ ($neg && !defined $ruleconf->{countries}{$cc}) ) {
+ dbg("$host ($ip) country $cc - HIT");
+ $pms->test_log("Host: $host in country $cc");
+ $pms->got_hit($rulename, "");
+ return 1;
+ } else {
+ dbg("$host ($ip) country $cc - ".($neg ? "excluded" : "no match"));
}
+ }
+ }
- if (defined $ruleconf->{continents}) {
- my $neg = defined $ruleconf->{continents_neg};
- my $testcont = join(' ', sort keys %{$ruleconf->{continents}});
- if ($neg) {
- dbg("checking $host for any continent except: $testcont");
- } else {
- dbg("checking $host for continents: $testcont");
- }
- foreach my $ip (@addrs) {
- my $cc = $geodb->get_continent($ip);
- if ( (!$neg && defined $ruleconf->{continents}{$cc}) ||
- ($neg && !defined $ruleconf->{continents}{$cc}) ) {
- dbg("$host ($ip) continent $cc - HIT");
- $pms->test_log("Host: $host in continent $cc");
- return 1; # hit
- } else {
- dbg("$host ($ip) continent $cc - ".($neg ? "excluded" : "no match"));
- }
- }
+ if (defined $ruleconf->{continents}) {
+ my $neg = defined $ruleconf->{continents_neg};
+ my $testcont = join(' ', sort keys %{$ruleconf->{continents}});
+ if ($neg) {
+ dbg("checking $host for any continent except: $testcont");
+ } else {
+ dbg("checking $host for continents: $testcont");
+ }
+ foreach my $ip (@$addrs) {
+ my $cc = $self->{geodb}->get_continent($ip);
+ if ( (!$neg && defined $ruleconf->{continents}{$cc}) ||
+ ($neg && !defined $ruleconf->{continents}{$cc}) ) {
+ dbg("$host ($ip) continent $cc - HIT");
+ $pms->test_log("Host: $host in continent $cc");
+ $pms->got_hit($rulename, "");
+ return 1;
+ } else {
+ dbg("$host ($ip) continent $cc - ".($neg ? "excluded" : "no match"));
}
+ }
+ }
- if (defined $ruleconf->{isps}) {
- if ($geodb->can('isp')) {
- my $testisp = join(', ', map {"\"$_\""} sort values %{$ruleconf->{isps}});
- dbg("checking $host for isps: $testisp");
-
- foreach my $ip (@addrs) {
- my $isp = $geodb->get_isp($ip);
- next unless defined $isp;
- my $ispkey = uc($isp); $ispkey =~ s/\s+//gs;
- if (defined $ruleconf->{isps}{$ispkey}) {
- dbg("$host ($ip) isp \"$isp\" - HIT");
- $pms->test_log("Host: $host in isp $isp");
- return 1; # hit
- } else {
- dbg("$host ($ip) isp $isp - no match");
- }
- }
+ if (defined $ruleconf->{isps}) {
+ if ($self->{geodb}->can('isp')) {
+ my $testisp = join(', ', map {"\"$_\""} sort values %{$ruleconf->{isps}});
+ dbg("checking $host for isps: $testisp");
+
+ foreach my $ip (@$addrs) {
+ my $isp = $self->{geodb}->get_isp($ip);
+ next unless defined $isp;
+ my $ispkey = uc($isp); $ispkey =~ s/\s+//gs;
+ if (defined $ruleconf->{isps}{$ispkey}) {
+ dbg("$host ($ip) isp \"$isp\" - HIT");
+ $pms->test_log("Host: $host in isp $isp");
+ $pms->got_hit($rulename, "");
+ return 1;
} else {
- dbg("skipping ISP check, GeoDB database not loaded");
+ dbg("$host ($ip) isp $isp - no match");
}
}
+ } else {
+ dbg("skipping ISP check, GeoDB database not loaded");
+ }
+ }
- if (defined $ruleconf->{netset}) {
- foreach my $ip (@addrs) {
- if ($ruleconf->{netset}->contains_ip($ip)) {
- dbg("$host ($ip) matches cidr - HIT");
- $pms->test_log("Host: $host in cidr");
- return 1; # hit
- } else {
- dbg("$host ($ip) not matching cidr");
- }
- }
+ if (defined $ruleconf->{netset}) {
+ foreach my $ip (@$addrs) {
+ if ($ruleconf->{netset}->contains_ip($ip)) {
+ dbg("$host ($ip) matches cidr - HIT");
+ $pms->test_log("Host: $host in cidr");
+ $pms->got_hit($rulename, "");
+ return 1;
+ } else {
+ dbg("$host ($ip) not matching cidr");
}
}
}
Added: spamassassin/trunk/t/data/spam/urilocalbl_net.eml
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/data/spam/urilocalbl_net.eml?rev=1842015&view=auto
==============================================================================
--- spamassassin/trunk/t/data/spam/urilocalbl_net.eml (added)
+++ spamassassin/trunk/t/data/spam/urilocalbl_net.eml Wed Sep 26 12:07:29 2018
@@ -0,0 +1,17 @@
+Received: from google-public-dns-a.google.com (google-public-dns-a.google.com [8.8.8.8])
+ by in.example.com (Postfix) with ESMTPS
+ for <te...@example.com>; Wed, 18 Jul 2018 21:12:22 +0200 (CEST)
+Received: by google-public-dns-a.google.com with SMTP id f21-v6so3811271wmc.5
+ for <te...@example.com>; Wed, 18 Jul 2018 12:12:22 -0700 (PDT)
+From: <te...@gmail.com>
+To: test@example.com
+Subject: Relay Country Test
+Date: Wed, 18 Jul 2018 12:12:00 -0700 (PDT)
+MIME-Version: 1.0
+Message-Id: <20...@gmail.com>
+
+<html>
+<body>
+<a href="https://google-public-dns-a.google.com/">IPv4 and IPv6</a>
+</body>
+</html>
Added: spamassassin/trunk/t/urilocalbl.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/urilocalbl.t?rev=1842015&view=auto
==============================================================================
--- spamassassin/trunk/t/urilocalbl.t (added)
+++ spamassassin/trunk/t/urilocalbl.t Wed Sep 26 12:07:29 2018
@@ -0,0 +1,194 @@
+#!/usr/bin/perl
+
+BEGIN {
+ if (-e 't/test_dir') { # if we are running "t/rule_tests.t", kluge around ...
+ chdir 't';
+ }
+
+ if (-e 'test_dir') { # running from test directory, not ..
+ unshift(@INC, '../blib/lib');
+ unshift(@INC, '../lib');
+ }
+}
+
+use lib '.'; use lib 't';
+use SATest; sa_t_init("urilocalbl");
+
+$tests = 0;
+eval { require GeoIP2::Database::Reader; $tests += 8; $has{GEOIP2} = 1 };
+eval { require Geo::IP; $tests += 8; $has{GEOIP} = 1 };
+eval { require IP::Country::DB_File; $tests += 8; $has{DB_FILE} = 1 };
+eval { require IP::Country::Fast; $tests += 8; $has{FAST} = 1 };
+
+use Test::More;
+
+plan skip_all => "No supported GeoDB module installed" unless $tests;
+
+$net = conf_bool('run_net_tests');
+$ipv6 = $net && conf_bool('run_ipv6_dns_tests');
+
+$tests *= 2 if $net;
+$tests += 1 if $ipv6 && defined $has{GEOIP2};
+$tests += 1 if $ipv6 && defined $has{DB_FILE};
+
+plan tests => $tests;
+
+# ---------------------------------------------------------------------------
+
+tstpre ("
+loadplugin Mail::SpamAssassin::Plugin::URILocalBL
+");
+
+%patterns_ipv4 = (
+ q{ X_URIBL_USA } => 'USA',
+ q{ X_URIBL_FINEG } => 'except Finland',
+ q{ X_URIBL_NA } => 'north America',
+ q{ X_URIBL_EUNEG } => 'except Europe',
+ q{ X_URIBL_CIDR1 } => 'our TestIP1',
+ q{ X_URIBL_CIDR2 } => 'our TestIP2',
+ q{ X_URIBL_CIDR3 } => 'our TestIP3',
+);
+
+%patterns_ipv6 = (
+ q{ X_URIBL_CIDR4 } => 'our TestIP4',
+);
+
+my $rules = "
+
+ uri_block_cc X_URIBL_USA us
+ describe X_URIBL_USA uri located in USA
+
+ uri_block_cc X_URIBL_FINEG !fi
+ describe X_URIBL_FINEG uri located anywhere except Finland
+
+ uri_block_cont X_URIBL_NA na
+ describe X_URIBL_NA uri located in north America
+
+ uri_block_cont X_URIBL_EUNEG !eu !af
+ describe X_URIBL_EUNEG uri located anywhere except Europe/Africa
+
+ uri_block_cidr X_URIBL_CIDR1 8.0.0.0/8 1.2.3.4
+ describe X_URIBL_CIDR1 uri is our TestIP1
+
+ uri_block_cidr X_URIBL_CIDR2 8.8.8.8
+ describe X_URIBL_CIDR2 uri is our TestIP2
+
+ uri_block_cidr X_URIBL_CIDR3 8.8.8.0/24
+ describe X_URIBL_CIDR3 uri is our TestIP3
+";
+
+my $rules_ipv6 = "
+
+ uri_block_cidr X_URIBL_CIDR4 2001:4860:4860::8888
+ describe X_URIBL_CIDR4 uri is our TestIP4
+";
+
+if (defined $has{GEOIP2}) {
+ my $lrules = "
+ geodb_module GeoIP2
+ geodb_search_path data/geodb
+ $rules
+ ";
+ tstlocalrules ($lrules);
+ %patterns = %patterns_ipv4;
+ ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+
+ if ($net) {
+ $lrules .= $rules_ipv6 if $ipv6;
+ tstlocalrules ($lrules);
+ if ($ipv6) {
+ %patterns = (%patterns_ipv4, %patterns_ipv6);
+ } else {
+ %patterns = %patterns_ipv4;
+ warn "skipping IPv6 DNS lookup tests (run_ipv6_dns_tests=n)\n";
+ }
+ ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+ } else {
+ warn "skipping DNS lookup tests (run_net_tests=n)\n";
+ }
+} else {
+ warn "skipping GeoIP2::Database::Reader tests (not installed)\n";
+}
+
+
+if (defined $has{GEOIP}) {
+ tstlocalrules ("
+ geodb_module Geo::IP
+ geodb_search_path data/geodb
+ $rules
+ ");
+ %patterns = %patterns_ipv4;
+ ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+
+ if ($net) {
+ ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+ } else {
+ warn "skipping DNS lookup tests (run_net_tests=n)\n";
+ }
+} else {
+ warn "skipping Geo::IP tests (not installed)\n";
+}
+
+
+if (defined $has{DB_FILE}) {
+ my $lrules = "
+ geodb_module DB_File
+ geodb_options country:data/geodb/ipcc.db
+ $rules
+ ";
+ tstlocalrules ($lrules);
+ %patterns = %patterns_ipv4;
+ ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+
+ if ($net) {
+ $lrules .= $rules_ipv6 if $ipv6;
+ tstlocalrules ($lrules);
+ if ($ipv6) {
+ %patterns = (%patterns_ipv4, %patterns_ipv6);
+ } else {
+ %patterns = %patterns_ipv4;
+ warn "skipping IPv6 DNS lookup tests (run_ipv6_dns_tests=n)\n";
+ }
+ ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+ } else {
+ warn "skipping DNS lookup tests (run_net_tests=n)\n";
+ }
+} else {
+ warn "skipping IP::Country::DB_File tests (not installed)\n";
+}
+
+
+if (defined $has{FAST}) {
+ tstlocalrules ("
+ geodb_module Fast
+ $rules
+ ");
+ %patterns = %patterns_ipv4;
+ ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+
+ if ($net) {
+ ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+ ok_all_patterns();
+ clear_pattern_counters();
+ } else {
+ warn "skipping DNS lookup tests (run_net_tests=n)\n";
+ }
+} else {
+ warn "skipping IP::Country::Fast tests (not installed)\n";
+}
+
+