You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by he...@apache.org on 2018/09/26 12:07:30 UTC

svn commit: r1842015 - in /spamassassin/trunk: MANIFEST lib/Mail/SpamAssassin/Plugin/URILocalBL.pm t/data/spam/urilocalbl_net.eml t/urilocalbl.t t/urilocalbl_dbfile.t t/urilocalbl_fast.t t/urilocalbl_geoip.t t/urilocalbl_geoip2.t

Author: hege
Date: Wed Sep 26 12:07:29 2018
New Revision: 1842015

URL: http://svn.apache.org/viewvc?rev=1842015&view=rev
Log:
Make URILocalBL use internal async DNS, make IPv6 lookups, combine tests to single file

Added:
    spamassassin/trunk/t/data/spam/urilocalbl_net.eml
    spamassassin/trunk/t/urilocalbl.t
Removed:
    spamassassin/trunk/t/urilocalbl_dbfile.t
    spamassassin/trunk/t/urilocalbl_fast.t
    spamassassin/trunk/t/urilocalbl_geoip.t
    spamassassin/trunk/t/urilocalbl_geoip2.t
Modified:
    spamassassin/trunk/MANIFEST
    spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm

Modified: spamassassin/trunk/MANIFEST
URL: http://svn.apache.org/viewvc/spamassassin/trunk/MANIFEST?rev=1842015&r1=1842014&r2=1842015&view=diff
==============================================================================
--- spamassassin/trunk/MANIFEST (original)
+++ spamassassin/trunk/MANIFEST Wed Sep 26 12:07:29 2018
@@ -558,10 +558,7 @@ t/uri_html.t
 t/uri_list.t
 t/uri_text.t
 t/uribl.t
-t/urilocalbl_dbfile.t
-t/urilocalbl_fast.t
-t/urilocalbl_geoip.t
-t/urilocalbl_geoip2.t
+t/urilocalbl.t
 t/utf8.t
 t/util_wrap.t
 t/whitelist_addrs.t

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm?rev=1842015&r1=1842014&r2=1842015&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URILocalBL.pm Wed Sep 26 12:07:29 2018
@@ -122,7 +122,10 @@ use version;
 
 our @ISA = qw(Mail::SpamAssassin::Plugin);
 
-sub dbg { Mail::SpamAssassin::Plugin::dbg ("URILocalBL: @_"); }
+sub dbg {
+  my $str = shift;
+  Mail::SpamAssassin::Plugin::dbg ("URILocalBL: $str", @_);
+}
 
 my $IP_ADDRESS = IP_ADDRESS;
 
@@ -331,11 +334,8 @@ sub finish_parsing_end {
   }
 }
 
-sub check_uri_local_bl {
+sub _init_geodb {
   my ($self, $pms) = @_;
-
-  return 0 if $self->{urilocalbl_disabled};
-
   if (!$self->{geodb}) {
     eval {
       $self->{geodb} = Mail::SpamAssassin::GeoDB->new({
@@ -349,14 +349,21 @@ sub check_uri_local_bl {
       return 0;
     }
   }
-  my $geodb = $self->{geodb};
+  return 1;
+}
+
+sub check_uri_local_bl {
+  my ($self, $pms) = @_;
+
+  return 0 if $self->{urilocalbl_disabled};
+  return 0 if !$self->_init_geodb($pms);
 
   my $rulename = $pms->get_current_eval_rule_name();
   my $ruleconf = $pms->{conf}->{urilocalbl}->{$rulename};
 
   dbg("running $rulename");
 
-  my @addrs;
+  my %found_hosts;
 
   foreach my $info (values %{$pms->get_uri_detail_list()}) {
     next unless $info->{hosts};
@@ -370,105 +377,160 @@ sub check_uri_local_bl {
         dbg("excluded $host, domain $domain matches");
         next HOST;
       }
-
-      if ($host !~ /^$IP_ADDRESS$/o) {
-        # this would be best cached from prior lookups
-        # TODO async extract_metadata lookup
-        @addrs = gethostbyname($host);
-        # convert to string values address list
-        @addrs = map { inet_ntoa($_); } @addrs[4..$#addrs];
-        if (@addrs) {
-          dbg("$host IP-addresses: ".join(', ', @addrs));
-        } else {
-          dbg("$host failed to resolve IP-addresses");
+      if ($host =~ /^$IP_ADDRESS$/o) {
+        if ($self->_check_host($pms, $rulename, $host, [$host])) {
+          # if hit, rule is done
+          return 0;
         }
       } else {
-        @addrs = ($host);
+        # do host lookups only after all IPs are checked, since they
+        # don't need resolving..
+        $found_hosts{$host} = 1;
       }
+    }
+  }
 
-      next HOST unless @addrs;
+  foreach my $host (keys %found_hosts) {
+    # launch dns
+    $pms->{async}->bgsend_and_start_lookup("$host.", 'A', undef,
+      { key => "urilocalbl:$host:A", host => $host, rulename => $rulename },
+      sub { my($ent, $pkt) = @_;
+            $self->_finish_lookup($pms, $ent, $pkt); },
+      master_deadline => $pms->{master_deadline}
+    );
+    # also IPv6 if database supports
+    $pms->{async}->bgsend_and_start_lookup("$host.", 'AAAA', undef,
+      { key => "urilocalbl:$host:AAAA", host => $host, rulename => $rulename },
+      sub { my($ent, $pkt) = @_;
+            $self->_finish_lookup($pms, $ent, $pkt); },
+      master_deadline => $pms->{master_deadline}
+    ) if $self->{geodb}->can('country_v6');
+  }
+}
 
-      foreach my $ip (@addrs) {
-        if (defined $ruleconf->{exclusions}{$ip}) {
-          dbg("excluded $host, IP $ip matches");
-          next HOST;
-        }
-      }
+sub _finish_lookup {
+  my ($self, $pms, $ent, $pkt) = @_;
 
-      if (defined $ruleconf->{countries}) {
-        my $neg = defined $ruleconf->{countries_neg};
-        my $testcc = join(' ', sort keys %{$ruleconf->{countries}});
-        if ($neg) {
-          dbg("checking $host for any country except: $testcc");
-        } else {
-          dbg("checking $host for countries: $testcc");
-        }
-        foreach my $ip (@addrs) {
-          my $cc = $geodb->get_country($ip);
-          if ( (!$neg && defined $ruleconf->{countries}{$cc}) ||
-               ($neg && !defined $ruleconf->{countries}{$cc}) ) {
-            dbg("$host ($ip) country $cc - HIT");
-            $pms->test_log("Host: $host in country $cc");
-            return 1; # hit
-          } else {
-            dbg("$host ($ip) country $cc - ".($neg ? "excluded" : "no match"));
-          }
-        }
+  my $rulename = $ent->{rulename};
+  my $host = $ent->{host};
+
+  # Skip duplicate A / AAAA matches
+  return if $pms->{urilocalbl_finished}->{$rulename};
+
+  if (!$pkt) {
+      # $pkt will be undef if the DNS query was aborted (e.g. timed out)
+      dbg("host lookup failed: $rulename $host");
+      return;
+  }
+
+  my @answer = $pkt->answer;
+  my @addrs;
+  foreach my $rr (@answer) {
+    if ($rr->type eq 'A' || $rr->type eq 'AAAA') {
+      push @addrs, $rr->address;
+    }
+  }
+
+  if (@addrs) {
+    if ($self->_check_host($pms, $rulename, $host, \@addrs)) {
+      $pms->register_async_rule_finish($rulename);
+      $pms->{urilocalbl_finished}->{$rulename} = 1;
+    }
+  }
+}
+
+sub _check_host {
+  my ($self, $pms, $rulename, $host, $addrs) = @_;
+
+  my $ruleconf = $pms->{conf}->{urilocalbl}->{$rulename};
+
+  if ($host ne $addrs->[0]) {
+    dbg("resolved $host: ".join(', ', @$addrs));
+  }
+
+  foreach my $ip (@$addrs) {
+    if (defined $ruleconf->{exclusions}{$ip}) {
+      dbg("excluded $host, IP $ip matches");
+      return 1;
+    }
+  }
+
+  if (defined $ruleconf->{countries}) {
+    my $neg = defined $ruleconf->{countries_neg};
+    my $testcc = join(' ', sort keys %{$ruleconf->{countries}});
+    if ($neg) {
+      dbg("checking $host for any country except: $testcc");
+    } else {
+      dbg("checking $host for countries: $testcc");
+    }
+    foreach my $ip (@$addrs) {
+      my $cc = $self->{geodb}->get_country($ip);
+      if ( (!$neg && defined $ruleconf->{countries}{$cc}) ||
+           ($neg && !defined $ruleconf->{countries}{$cc}) ) {
+        dbg("$host ($ip) country $cc - HIT");
+        $pms->test_log("Host: $host in country $cc");
+        $pms->got_hit($rulename, "");
+        return 1;
+      } else {
+        dbg("$host ($ip) country $cc - ".($neg ? "excluded" : "no match"));
       }
+    }
+  }
 
-      if (defined $ruleconf->{continents}) {
-        my $neg = defined $ruleconf->{continents_neg};
-        my $testcont = join(' ', sort keys %{$ruleconf->{continents}});
-        if ($neg) {
-          dbg("checking $host for any continent except: $testcont");
-        } else {
-          dbg("checking $host for continents: $testcont");
-        }
-        foreach my $ip (@addrs) {
-          my $cc = $geodb->get_continent($ip);
-          if ( (!$neg && defined $ruleconf->{continents}{$cc}) ||
-               ($neg && !defined $ruleconf->{continents}{$cc}) ) {
-            dbg("$host ($ip) continent $cc - HIT");
-            $pms->test_log("Host: $host in continent $cc");
-            return 1; # hit
-          } else {
-            dbg("$host ($ip) continent $cc - ".($neg ? "excluded" : "no match"));
-          }
-        }
+  if (defined $ruleconf->{continents}) {
+    my $neg = defined $ruleconf->{continents_neg};
+    my $testcont = join(' ', sort keys %{$ruleconf->{continents}});
+    if ($neg) {
+      dbg("checking $host for any continent except: $testcont");
+    } else {
+      dbg("checking $host for continents: $testcont");
+    }
+    foreach my $ip (@$addrs) {
+      my $cc = $self->{geodb}->get_continent($ip);
+      if ( (!$neg && defined $ruleconf->{continents}{$cc}) ||
+           ($neg && !defined $ruleconf->{continents}{$cc}) ) {
+        dbg("$host ($ip) continent $cc - HIT");
+        $pms->test_log("Host: $host in continent $cc");
+        $pms->got_hit($rulename, "");
+        return 1;
+      } else {
+        dbg("$host ($ip) continent $cc - ".($neg ? "excluded" : "no match"));
       }
+    }
+  }
 
-      if (defined $ruleconf->{isps}) {
-        if ($geodb->can('isp')) {
-          my $testisp = join(', ', map {"\"$_\""} sort values %{$ruleconf->{isps}});
-          dbg("checking $host for isps: $testisp");
-
-          foreach my $ip (@addrs) {
-            my $isp = $geodb->get_isp($ip);
-            next unless defined $isp;
-            my $ispkey = uc($isp); $ispkey =~ s/\s+//gs;
-            if (defined $ruleconf->{isps}{$ispkey}) {
-              dbg("$host ($ip) isp \"$isp\" - HIT");
-              $pms->test_log("Host: $host in isp $isp");
-              return 1; # hit
-            } else {
-              dbg("$host ($ip) isp $isp - no match");
-            }
-          }
+  if (defined $ruleconf->{isps}) {
+    if ($self->{geodb}->can('isp')) {
+      my $testisp = join(', ', map {"\"$_\""} sort values %{$ruleconf->{isps}});
+      dbg("checking $host for isps: $testisp");
+
+      foreach my $ip (@$addrs) {
+        my $isp = $self->{geodb}->get_isp($ip);
+        next unless defined $isp;
+        my $ispkey = uc($isp); $ispkey =~ s/\s+//gs;
+        if (defined $ruleconf->{isps}{$ispkey}) {
+          dbg("$host ($ip) isp \"$isp\" - HIT");
+          $pms->test_log("Host: $host in isp $isp");
+          $pms->got_hit($rulename, "");
+          return 1;
         } else {
-          dbg("skipping ISP check, GeoDB database not loaded");
+          dbg("$host ($ip) isp $isp - no match");
         }
       }
+    } else {
+      dbg("skipping ISP check, GeoDB database not loaded");
+    }
+  }
 
-      if (defined $ruleconf->{netset}) {
-        foreach my $ip (@addrs) {
-          if ($ruleconf->{netset}->contains_ip($ip)) {
-            dbg("$host ($ip) matches cidr - HIT");
-            $pms->test_log("Host: $host in cidr");
-            return 1; # hit
-          } else {
-            dbg("$host ($ip) not matching cidr");
-          }
-        }
+  if (defined $ruleconf->{netset}) {
+    foreach my $ip (@$addrs) {
+      if ($ruleconf->{netset}->contains_ip($ip)) {
+        dbg("$host ($ip) matches cidr - HIT");
+        $pms->test_log("Host: $host in cidr");
+        $pms->got_hit($rulename, "");
+        return 1;
+      } else {
+        dbg("$host ($ip) not matching cidr");
       }
     }
   }

Added: spamassassin/trunk/t/data/spam/urilocalbl_net.eml
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/data/spam/urilocalbl_net.eml?rev=1842015&view=auto
==============================================================================
--- spamassassin/trunk/t/data/spam/urilocalbl_net.eml (added)
+++ spamassassin/trunk/t/data/spam/urilocalbl_net.eml Wed Sep 26 12:07:29 2018
@@ -0,0 +1,17 @@
+Received: from google-public-dns-a.google.com (google-public-dns-a.google.com [8.8.8.8])
+	by in.example.com (Postfix) with ESMTPS
+	for <te...@example.com>; Wed, 18 Jul 2018 21:12:22 +0200 (CEST)
+Received: by google-public-dns-a.google.com with SMTP id f21-v6so3811271wmc.5
+        for <te...@example.com>; Wed, 18 Jul 2018 12:12:22 -0700 (PDT)
+From: <te...@gmail.com>
+To: test@example.com
+Subject: Relay Country Test
+Date: Wed, 18 Jul 2018 12:12:00 -0700 (PDT)
+MIME-Version: 1.0
+Message-Id: <20...@gmail.com>
+
+<html>
+<body>
+<a href="https://google-public-dns-a.google.com/">IPv4 and IPv6</a>
+</body>
+</html>

Added: spamassassin/trunk/t/urilocalbl.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/urilocalbl.t?rev=1842015&view=auto
==============================================================================
--- spamassassin/trunk/t/urilocalbl.t (added)
+++ spamassassin/trunk/t/urilocalbl.t Wed Sep 26 12:07:29 2018
@@ -0,0 +1,194 @@
+#!/usr/bin/perl
+
+BEGIN {
+  if (-e 't/test_dir') { # if we are running "t/rule_tests.t", kluge around ...
+    chdir 't';
+  }
+
+  if (-e 'test_dir') {            # running from test directory, not ..
+    unshift(@INC, '../blib/lib');
+    unshift(@INC, '../lib');
+  }
+}
+
+use lib '.'; use lib 't';
+use SATest; sa_t_init("urilocalbl");
+
+$tests = 0;
+eval { require GeoIP2::Database::Reader; $tests += 8; $has{GEOIP2}  = 1 };
+eval { require Geo::IP;                  $tests += 8; $has{GEOIP}   = 1 };
+eval { require IP::Country::DB_File;     $tests += 8; $has{DB_FILE} = 1 };
+eval { require IP::Country::Fast;        $tests += 8; $has{FAST}    = 1 };
+
+use Test::More;
+
+plan skip_all => "No supported GeoDB module installed" unless $tests;
+
+$net = conf_bool('run_net_tests');
+$ipv6 = $net && conf_bool('run_ipv6_dns_tests');
+
+$tests *= 2 if $net;
+$tests += 1 if $ipv6 && defined $has{GEOIP2};
+$tests += 1 if $ipv6 && defined $has{DB_FILE};
+
+plan tests => $tests;
+
+# ---------------------------------------------------------------------------
+
+tstpre ("
+loadplugin Mail::SpamAssassin::Plugin::URILocalBL
+");
+
+%patterns_ipv4 = (
+  q{ X_URIBL_USA } => 'USA',
+  q{ X_URIBL_FINEG } => 'except Finland',
+  q{ X_URIBL_NA } => 'north America',
+  q{ X_URIBL_EUNEG } => 'except Europe',
+  q{ X_URIBL_CIDR1 } => 'our TestIP1',
+  q{ X_URIBL_CIDR2 } => 'our TestIP2',
+  q{ X_URIBL_CIDR3 } => 'our TestIP3',
+);
+
+%patterns_ipv6 = (
+  q{ X_URIBL_CIDR4 } => 'our TestIP4',
+);
+
+my $rules = "
+
+  uri_block_cc X_URIBL_USA us
+  describe X_URIBL_USA uri located in USA
+  
+  uri_block_cc X_URIBL_FINEG !fi
+  describe X_URIBL_FINEG uri located anywhere except Finland
+
+  uri_block_cont X_URIBL_NA na
+  describe X_URIBL_NA uri located in north America
+
+  uri_block_cont X_URIBL_EUNEG !eu !af
+  describe X_URIBL_EUNEG uri located anywhere except Europe/Africa
+
+  uri_block_cidr X_URIBL_CIDR1 8.0.0.0/8 1.2.3.4
+  describe X_URIBL_CIDR1 uri is our TestIP1
+
+  uri_block_cidr X_URIBL_CIDR2 8.8.8.8
+  describe X_URIBL_CIDR2 uri is our TestIP2
+
+  uri_block_cidr X_URIBL_CIDR3 8.8.8.0/24
+  describe X_URIBL_CIDR3 uri is our TestIP3
+";
+
+my $rules_ipv6 = "
+
+  uri_block_cidr X_URIBL_CIDR4 2001:4860:4860::8888
+  describe X_URIBL_CIDR4 uri is our TestIP4
+";
+
+if (defined $has{GEOIP2}) {
+  my $lrules = "
+    geodb_module GeoIP2
+    geodb_search_path data/geodb
+    $rules
+  ";
+  tstlocalrules ($lrules);
+  %patterns = %patterns_ipv4;
+  ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+  ok_all_patterns();
+  clear_pattern_counters();
+
+  if ($net) {
+    $lrules .= $rules_ipv6 if $ipv6;
+    tstlocalrules ($lrules);
+    if ($ipv6) {
+      %patterns = (%patterns_ipv4, %patterns_ipv6);
+    } else {
+      %patterns = %patterns_ipv4;
+      warn "skipping IPv6 DNS lookup tests (run_ipv6_dns_tests=n)\n";
+    }
+    ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+    ok_all_patterns();
+    clear_pattern_counters();
+  } else {
+    warn "skipping DNS lookup tests (run_net_tests=n)\n";
+  }
+} else {
+  warn "skipping GeoIP2::Database::Reader tests (not installed)\n";
+}
+
+
+if (defined $has{GEOIP}) {
+  tstlocalrules ("
+    geodb_module Geo::IP
+    geodb_search_path data/geodb
+    $rules
+  ");
+  %patterns = %patterns_ipv4;
+  ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+  ok_all_patterns();
+  clear_pattern_counters();
+
+  if ($net) {
+    ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+    ok_all_patterns();
+    clear_pattern_counters();
+  } else {
+    warn "skipping DNS lookup tests (run_net_tests=n)\n";
+  }
+} else {
+  warn "skipping Geo::IP tests (not installed)\n";
+}
+
+
+if (defined $has{DB_FILE}) {
+  my $lrules = "
+    geodb_module DB_File
+    geodb_options country:data/geodb/ipcc.db
+    $rules
+  ";
+  tstlocalrules ($lrules);
+  %patterns = %patterns_ipv4;
+  ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+  ok_all_patterns();
+  clear_pattern_counters();
+
+  if ($net) {
+    $lrules .= $rules_ipv6 if $ipv6;
+    tstlocalrules ($lrules);
+    if ($ipv6) {
+      %patterns = (%patterns_ipv4, %patterns_ipv6);
+    } else {
+      %patterns = %patterns_ipv4;
+      warn "skipping IPv6 DNS lookup tests (run_ipv6_dns_tests=n)\n";
+    }
+    ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+    ok_all_patterns();
+    clear_pattern_counters();
+  } else {
+    warn "skipping DNS lookup tests (run_net_tests=n)\n";
+  }
+} else {
+  warn "skipping IP::Country::DB_File tests (not installed)\n";
+}
+
+
+if (defined $has{FAST}) {
+  tstlocalrules ("
+    geodb_module Fast
+    $rules
+  ");
+  %patterns = %patterns_ipv4;
+  ok sarun ("-L -t < data/spam/relayUS.eml", \&patterns_run_cb);
+  ok_all_patterns();
+  clear_pattern_counters();
+
+  if ($net) {
+    ok sarun ("-L -t < data/spam/urilocalbl_net.eml", \&patterns_run_cb);
+    ok_all_patterns();
+    clear_pattern_counters();
+  } else {
+    warn "skipping DNS lookup tests (run_net_tests=n)\n";
+  }
+} else {
+  warn "skipping IP::Country::Fast tests (not installed)\n";
+}
+
+