You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by do...@apache.org on 2006/12/13 03:25:21 UTC

svn commit: r486461 - in /spamassassin/trunk: INSTALL lib/Mail/SpamAssassin/DnsResolver.pm lib/Mail/SpamAssassin/Plugin/SPF.pm t/spf.t

Author: dos
Date: Tue Dec 12 18:25:20 2006
New Revision: 486461

URL: http://svn.apache.org/viewvc?view=rev&rev=486461
Log:
bug 5236: Support Mail::SPF replacement for Mail::SPF::Query

  - adds support for Mail::SPF while maintaining fall back support for
    Mail::SPF::Query; Mail::SPF is the current reference implementation for
    RFC 4408, it omits a lot of legacy crud present in Mail::SPF::Query

  - minor optimizations to and removal of crud from the SPF plugin

  - removes failure comments (for test logs) for all but mfrom or helo FAIL
    results per RFC 4408 section 6.2

  - handles some previously unchecked undef values in debugs

  - adds tests to t/spf.t for spf based whitelisting (previously untested)

  - modified t/spf.t to test both of the supported SPF modules specifically

  - adds an exteremely minimal errorstring() method to M::SA::DnsResolver
    so that Mail::SPF can use M::SA::DnsResolver instead of its own
    Net::DNS::Resolver object


Modified:
    spamassassin/trunk/INSTALL
    spamassassin/trunk/lib/Mail/SpamAssassin/DnsResolver.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/SPF.pm
    spamassassin/trunk/t/spf.t

Modified: spamassassin/trunk/INSTALL
URL: http://svn.apache.org/viewvc/spamassassin/trunk/INSTALL?view=diff&rev=486461&r1=486460&r2=486461
==============================================================================
--- spamassassin/trunk/INSTALL (original)
+++ spamassassin/trunk/INSTALL Tue Dec 12 18:25:20 2006
@@ -253,11 +253,15 @@
     Used when manually reporting spam to SpamCop.
 
 
-  - Mail::SPF::Query (from CPAN)
+  - Mail::SPF (from CPAN) or Mail::SPF::Query (from CPAN)
 
     Used to check DNS Sender Policy Framework (SPF) records to fight email
     address forgery and make it easier to identify spams.
 
+    Either of Mail::SPF or Mail::SPF::Query can be used but Mail::SPF is
+    preferred as it is the current reference implementation for RFC 4408.
+
+    Net::DNS version 0.58 or higher is required to use Mail::SPF.
     Net::DNS version 0.34 or higher is required to use Mail::SPF::Query.
 
 

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/DnsResolver.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/DnsResolver.pm?view=diff&rev=486461&r1=486460&r2=486461
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/DnsResolver.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/DnsResolver.pm Tue Dec 12 18:25:20 2006
@@ -357,6 +357,8 @@
   my ($self, $host, $type, $class, $cb) = @_;
   return if $self->{no_resolver};
 
+  $self->{send_timed_out} = 0;
+
   my $pkt = $self->new_dns_packet($host, $type, $class);
 
   $self->connect_sock_if_reqd();
@@ -470,10 +472,32 @@
 
     while (($now < $deadline) && (!defined($answerpkt))) {
       $self->poll_responses(1);
+      last if defined $answerpkt;
       $now = time;
     }
+    $self->{send_timed_out} = 1 unless ($now < $deadline);
   }
   return $answerpkt;
+}
+
+###########################################################################
+
+=item $res->errorstring()
+
+Little more than a stub for callers expecting this from C<Net::DNS::Resolver>.
+
+If called immediately after a call to $res->send this will return
+C<query timed out> if the $res->send DNS query timed out.  Otherwise 
+C<unknown error or no error> will be returned.
+
+No other errors are reported.
+
+=cut
+
+sub errorstring {
+  my ($self) = @_;
+  return 'query timed out' if $self->{send_timed_out};
+  return 'unknown error or no error';
 }
 
 ###########################################################################

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/SPF.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/SPF.pm?view=diff&rev=486461&r1=486460&r2=486461
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/SPF.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/SPF.pm Tue Dec 12 18:25:20 2006
@@ -53,8 +53,6 @@
   my $self = $class->SUPER::new($mailsaobject);
   bless ($self, $class);
 
-  my $conf = $mailsaobject->{conf};
-
   $self->register_eval_rule ("check_for_spf_pass");
   $self->register_eval_rule ("check_for_spf_neutral");
   $self->register_eval_rule ("check_for_spf_fail");
@@ -133,6 +131,44 @@
     type => $Mail::SpamAssassin::Conf::CONF_TYPE_ADDRLIST
   });
 
+=back
+
+=head1 ADMINISTRATOR OPTIONS
+
+=over 4
+
+=item do_not_use_mail_spf (0|1)		(default: 0)
+
+By default the plugin will try to use the Mail::SPF module for SPF checks if
+it can be loaded.  If Mail::SPF cannot be used the plugin will fall back to
+using the legacy Mail::SPF::Query module if it can be loaded.
+
+Use this option to stop the plugin from using Mail::SPF and cause it to try to
+use Mail::SPF::Query instead.
+
+=cut
+
+  push(@cmds, {
+    setting => 'do_not_use_mail_spf',
+    is_admin => 1,
+    default => 0,
+    type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL,
+  });
+
+=item do_not_use_mail_spq_query (0|1)	(default: 0)
+
+As above, but instead stop the plugin from trying to use Mail::SPF::Query and
+cause it to only try to use Mail::SPF.
+
+=cut
+
+  push(@cmds, {
+    setting => 'do_not_use_mail_spf_query',
+    is_admin => 1,
+    default => 0,
+    type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL,
+  });
+
   $conf->{parser}->register_commands(\@cmds);
 }
 
@@ -146,9 +182,6 @@
 sub check_for_spf_neutral {
   my ($self, $scanner) = @_;
   $self->_check_spf ($scanner, 0) unless $scanner->{spf_checked};
-  if ($scanner->{spf_failure_comment}) {
-    $scanner->test_log ($scanner->{spf_failure_comment});
-  }
   $scanner->{spf_neutral};
 }
 
@@ -164,9 +197,6 @@
 sub check_for_spf_softfail {
   my ($self, $scanner) = @_;
   $self->_check_spf ($scanner, 0) unless $scanner->{spf_checked};
-  if ($scanner->{spf_failure_comment}) {
-    $scanner->test_log ($scanner->{spf_failure_comment});
-  }
   $scanner->{spf_softfail};
 }
 
@@ -179,9 +209,6 @@
 sub check_for_spf_helo_neutral {
   my ($self, $scanner) = @_;
   $self->_check_spf ($scanner, 1) unless $scanner->{spf_helo_checked};
-  if ($scanner->{spf_helo_failure_comment}) {
-    $scanner->test_log ($scanner->{spf_helo_failure_comment});
-  }
   $scanner->{spf_helo_neutral};
 }
 
@@ -197,9 +224,6 @@
 sub check_for_spf_helo_softfail {
   my ($self, $scanner) = @_;
   $self->_check_spf ($scanner, 1) unless $scanner->{spf_helo_checked};
-  if ($scanner->{spf_helo_failure_comment}) {
-    $scanner->test_log ($scanner->{spf_helo_failure_comment});
-  }
   $scanner->{spf_helo_softfail};
 }
 
@@ -219,6 +243,56 @@
   my ($self, $scanner, $ishelo) = @_;
 
   return unless $scanner->is_dns_available();
+  return if $self->{no_spf_module};
+
+  # select the SPF module we're going to use
+  unless (defined $self->{has_mail_spf}) {
+    eval {
+      die("Mail::SPF disabled by admin setting\n") if $scanner->{conf}->{do_not_use_mail_spf};
+
+      require Mail::SPF;
+      if (!defined $Mail::SPF::VERSION || $Mail::SPF::VERSION < 2.001) {
+	die "Mail::SPF 2.001 or later required, this is ".
+	  (defined $Mail::SPF::VERSION ? $Mail::SPF::VERSION : 'unknown')."\n";
+      }
+      # Mail::SPF::Server can be re-used, and we get to use our own resolver object!
+      $self->{spf_server} = Mail::SPF::Server->new(
+				hostname     => $scanner->get_tag('HOSTNAME'),
+				dns_resolver => $self->{main}->{resolver} );
+    };
+
+    unless ($@) {
+      dbg("spf: using Mail::SPF for SPF checks");
+      $self->{has_mail_spf} = 1;
+    } else {
+      # strip the @INC paths... users are going to see it and think there's a problem even though
+      # we're going to fall back to Mail::SPF::Query (which will display the same paths if it fails)
+      $@ =~ s#^Can't locate Mail/SPFd.pm in \@INC .*#Can't locate Mail/SPFd.pm#;
+      dbg("spf: cannot load Mail::SPF module or create Mail::SPF::Server object: $@");
+      dbg("spf: attempting to use legacy Mail::SPF::Query module instead");
+
+      eval {
+	die("Mail::SPF::Query disabled by admin setting\n") if $scanner->{conf}->{do_not_use_mail_spf_query};
+
+	require Mail::SPF::Query;
+	if (!defined $Mail::SPF::Query::VERSION || $Mail::SPF::Query::VERSION < 1.996) {
+	  die "Mail::SPF::Query 1.996 or later required, this is ".
+	    (defined $Mail::SPF::Query::VERSION ? $Mail::SPF::Query::VERSION : 'unknown')."\n";
+	}
+      };
+
+      unless ($@) {
+	dbg("spf: using Mail::SPF::Query for SPF checks");
+	$self->{has_mail_spf} = 0;
+      } else {
+	dbg("spf: cannot load Mail::SPF::Query module: $@");
+	dbg("spf: one of Mail::SPF or Mail::SPF::Query is required for SPF checks, SPF checks disabled");
+	$self->{no_spf_module} = 1;
+	return;
+      }
+    }
+  }
+
 
   # skip SPF checks if the A/MX records are nonexistent for the From
   # domain, anyway, to avoid crappy messages from slowing us down
@@ -249,21 +323,29 @@
     return;
   }
 
-  my $ip = $lasthop->{ip};
-  my $helo = $lasthop->{helo};
+  my $ip = $lasthop->{ip};	# always present
+  my $helo = $lasthop->{helo};	# could be missing
   $scanner->{sender} = '' unless $scanner->{sender_got};
 
   if ($ishelo) {
+    unless ($helo) {
+      dbg("spf: cannot check HELO, HELO value unknown");
+      return;
+    }
     dbg("spf: checking HELO (helo=$helo, ip=$ip)");
-
   } else {
     $self->_get_sender($scanner) unless $scanner->{sender_got};
 
+    # TODO: we're supposed to use the helo domain as the sender identity (for
+    # mfrom checks) if the sender is the null sender, however determining that
+    # it's the null sender, and not just a failure to get the envelope isn't
+    # exactly trivial... so for now we'll just skip the check
+
     if (!$scanner->{sender}) {
       # we already dbg'd that we couldn't get an Envelope-From and can't do SPF
       return;
     }
-    dbg("spf: checking EnvelopeFrom (helo=$helo, ip=$ip, envfrom=$scanner->{sender})");
+    dbg("spf: checking EnvelopeFrom (helo=".($helo ? $helo : '').", ip=$ip, envfrom=$scanner->{sender})");
   }
 
   # this test could probably stand to be more strict, but try to test
@@ -272,43 +354,88 @@
     dbg("spf: cannot check HELO of '$helo', skipping");
     return;
   }
-  if (!$helo) {
-    dbg("spf: cannot get HELO, cannot use SPF");
-    return;
-  }
 
-  if ($scanner->server_failed_to_respond_for_domain($helo)) {
+  if ($helo && $scanner->server_failed_to_respond_for_domain($helo)) {
     dbg("spf: we had a previous timeout on '$helo', skipping");
     return;
   }
 
-  my $query;
-  eval {
-    require Mail::SPF::Query;
-    if (!defined $Mail::SPF::Query::VERSION || $Mail::SPF::Query::VERSION < 1.996) {
-      die "spf: Mail::SPF::Query 1.996 or later required, this is $Mail::SPF::Query::VERSION\n";
+
+  my ($result, $comment, $text, $err);
+
+  # use Mail::SPF if it was available, otherwise use the legacy Mail::SPF::Query
+  if ($self->{has_mail_spf}) {
+
+    # TODO: currently we won't get to here for a mfrom check with a null sender
+    my $identity = $ishelo ? $helo : ($scanner->{sender}); # || $helo);
+
+    unless ($identity) {
+      dbg("spf: cannot determine ".($ishelo ? 'helo' : 'mfrom').
+	  " identity, skipping ".($ishelo ? 'helo' : 'mfrom')." SPF check");
+      return;
+    }
+    $helo ||= 'unknown';  # only used for macro expansion in the mfrom explanation
+
+    my $request;
+    eval {
+      $request = Mail::SPF::Request->new( scope         => $ishelo ? 'helo' : 'mfrom',
+					  identity      => $identity,
+					  ip_address    => $ip,
+					  helo_identity => $helo );
+    };
+
+    if ($@) {
+      dbg("spf: cannot create Mail::SPF::Request object");
+      return;
+    }
+
+    my $timeout = $scanner->{conf}->{spf_timeout};
+
+    my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout });
+    $err = $timer->run_and_catch(sub {
+
+      my $query = $self->{spf_server}->process($request);
+
+      $result = $query->code;
+      $comment = $query->authority_explanation if $query->can("authority_explanation");
+      $text = $query->text;
+
+    });
+
+
+  } else {
+
+    if (!$helo) {
+      dbg("spf: cannot get HELO, cannot use Mail::SPF::Query, consider installing Mail::SPF");
+      return;
     }
-    $query = Mail::SPF::Query->new (ip => $ip,
+
+    # TODO: if we start doing checks on the null sender using the helo domain
+    # be sure to fix this so that it uses the correct sender identity
+    my $query;
+    eval {
+      $query = Mail::SPF::Query->new (ip => $ip,
 				    sender => $scanner->{sender},
 				    helo => $helo,
 				    debug => 0,
 				    trusted => 0);
-  };
+    };
 
-  if ($@) {
-    dbg("spf: cannot load or create Mail::SPF::Query module: $@");
-    return;
-  }
+    if ($@) {
+      dbg("spf: cannot create Mail::SPF::Query object: $@");
+      return;
+    }
 
-  my ($result, $comment);
-  my $timeout = $scanner->{conf}->{spf_timeout};
+    my $timeout = $scanner->{conf}->{spf_timeout};
 
-  my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout });
-  my $err = $timer->run_and_catch(sub {
+    my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout });
+    $err = $timer->run_and_catch(sub {
 
-    ($result, $comment) = $query->result();
+      ($result, $comment) = $query->result();
 
-  });
+    });
+
+  } # end of differences between Mail::SPF and Mail::SPF::Query
 
   if ($err) {
     chomp $err;
@@ -316,9 +443,12 @@
     return 0;
   }
 
+
   $result ||= 'timeout';	# bug 5077
   $comment ||= '';
   $comment =~ s/\s+/ /gs;	# no newlines please
+  $text ||= '';
+  $text =~ s/\s+/ /gs;		# no newlines please
 
   if ($ishelo) {
     if ($result eq 'pass') { $scanner->{spf_helo_pass} = 1; }
@@ -326,7 +456,7 @@
     elsif ($result eq 'fail') { $scanner->{spf_helo_fail} = 1; }
     elsif ($result eq 'softfail') { $scanner->{spf_helo_softfail} = 1; }
 
-    if ($result eq 'neutral' || $result eq 'fail' || $result eq 'softfail') {
+    if ($result eq 'fail') {	# RFC 4408 6.2
       $scanner->{spf_helo_failure_comment} = "SPF failed: $comment";
     }
   } else {
@@ -335,12 +465,12 @@
     elsif ($result eq 'fail') { $scanner->{spf_fail} = 1; }
     elsif ($result eq 'softfail') { $scanner->{spf_softfail} = 1; }
 
-    if ($result eq 'neutral' || $result eq 'fail' || $result eq 'softfail') {
+    if ($result eq 'fail') {	# RCF 4408 6.2
       $scanner->{spf_failure_comment} = "SPF failed: $comment";
     }
   }
 
-  dbg("spf: query for $scanner->{sender}/$ip/$helo: result: $result, comment: $comment");
+  dbg("spf: query for $scanner->{sender}/$ip/$helo: result: $result, comment: $comment, text: $text");
 }
 
 sub _get_relay {
@@ -392,10 +522,18 @@
   my ($self, $scanner) = @_;
 
   return unless $scanner->is_dns_available();
+  return if ($self->{no_spf_module});
 
   $scanner->{spf_whitelist_from_checked} = 1;
   $scanner->{spf_whitelist_from} = 0;
 
+  # if we've already checked for an SPF PASS and didn't get it don't waste time
+  # checking to see if the sender address is in the spf whitelist
+  if ($scanner->{spf_checked} && !$scanner->{spf_pass}) {
+    dbg("spf: whitelist_from_spf: already checked spf and didn't get pass, skipping whitelist check");
+    return;
+  }
+
   $self->_get_sender($scanner) unless $scanner->{sender_got};
 
   unless ($scanner->{sender}) {
@@ -432,9 +570,17 @@
   my ($self, $scanner) = @_;
 
   return unless $scanner->is_dns_available();
+  return if ($self->{no_spf_module});
 
   $scanner->{def_spf_whitelist_from_checked} = 1;
   $scanner->{def_spf_whitelist_from} = 0;
+
+  # if we've already checked for an SPF PASS and didn't get it don't waste time
+  # checking to see if the sender address is in the spf whitelist
+  if ($scanner->{spf_checked} && !$scanner->{spf_pass}) {
+    dbg("spf: def_spf_whitelist_from: already checked spf and didn't get pass, skipping whitelist check");
+    return;
+  }
 
   $self->_get_sender($scanner) unless $scanner->{sender_got};
 

Modified: spamassassin/trunk/t/spf.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/spf.t?view=diff&rev=486461&r1=486460&r2=486461
==============================================================================
--- spamassassin/trunk/t/spf.t (original)
+++ spamassassin/trunk/t/spf.t Tue Dec 12 18:25:20 2006
@@ -6,6 +6,7 @@
 
 use constant TEST_ENABLED => conf_bool('run_net_tests');
 use constant HAS_SPFQUERY => eval { require Mail::SPF::Query; };
+use constant HAS_MAILSPF => eval { require Mail::SPF; };
 # bug 3806:
 # Do not run this test on non-Linux unices as root, due to a bug
 # in Sys::Hostname::Long (which Mail::Query::SPF uses.)
@@ -13,13 +14,14 @@
 use constant IS_WINDOWS => ($^O =~ /^(mswin|dos|os2)/oi);
 use constant AM_ROOT    => $< == 0;
 
-use constant DO_RUN     => TEST_ENABLED && HAS_SPFQUERY &&
+use constant DO_RUN     => TEST_ENABLED && (HAS_SPFQUERY || HAS_MAILSPF) &&
                                         !(AM_ROOT &&
                                           !(IS_LINUX || IS_WINDOWS));
 
 BEGIN {
-  
-  plan tests => (DO_RUN ? 36 : 0);
+
+  # some tests are run once for each SPF module, others are only run once
+  plan tests => (DO_RUN ? (HAS_SPFQUERY && HAS_MAILSPF ? 90 : (HAS_SPFQUERY ? 46 : 46)) : 0);
 
 };
 
@@ -37,156 +39,175 @@
   score SPF_SOFTFAIL 0.001
   score SPF_PASS -0.001
   score SPF_HELO_PASS -0.001
+  score USER_IN_DEF_SPF_WL -0.001
+  score USER_IN_SPF_WHITELIST -0.001
 ");
 
-%patterns = (
+# test both of the SPF modules we support
+for $disable_an_spf_module ('do_not_use_mail_spf 1', 'do_not_use_mail_spf_query 1') {
+
+  # only do the tests if the module that wasn't disabled is available
+  next if ($disable_an_spf_module eq 'do_not_use_mail_spf 1' && !HAS_SPFQUERY);
+  next if ($disable_an_spf_module eq 'do_not_use_mail_spf_query 1' && !HAS_MAILSPF);
+
+  tstprefs("
+    $disable_an_spf_module
+  ");
+
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf1", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf1", \&patterns_run_cb);
+  ok_all_patterns();
 
-%patterns = (
+  %patterns = (
     q{ SPF_NEUTRAL }, 'neutral',
     q{ SPF_HELO_NEUTRAL }, 'helo_neutral',
-);
+  );
 
-sarun ("-t < data/spam/spf1", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/spam/spf1", \&patterns_run_cb);
+  ok_all_patterns();
 
-%patterns = (
+  %patterns = (
     q{ SPF_SOFTFAIL }, 'softfail',
     q{ SPF_HELO_SOFTFAIL }, 'helo_softfail',
-);
+  );
 
-sarun ("-t < data/spam/spf2", \&patterns_run_cb);
-ok_all_patterns();
-%patterns = (
+  sarun ("-t < data/spam/spf2", \&patterns_run_cb);
+  ok_all_patterns();
+  %patterns = (
     q{ SPF_FAIL }, 'fail',
     q{ SPF_HELO_FAIL }, 'helo_fail',
-);
+  );
 
-sarun ("-t < data/spam/spf3", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/spam/spf3", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# Test using an assortment of trusted and internal network definitions
+  # Test using an assortment of trusted and internal network definitions
 
-# 9-10: Trusted networks contain first header.
+  # 9-10: Trusted networks contain first header.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.157
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.157
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%patterns = (
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 11-12: Internal networks contain first header.
-#	 Trusted networks not defined.
+  # 11-12: Internal networks contain first header.
+  #	   Trusted networks not defined.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-internal_networks 65.214.43.157
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    internal_networks 65.214.43.157
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%patterns = (
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
-
-
-# 13-14: Internal networks contain first header.
-#	 Trusted networks contain some other IP.
-#      jm: commented; this is now an error condition.
-
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 1.2.3.4
-internal_networks 65.214.43.157
-always_trust_envelope_sender 1
-");
-
-%patterns = (
-  q{ SPF_HELO_NEUTRAL }, 'helo_neutral',
-  q{ SPF_NEUTRAL }, 'neutral',
-);
-
-if (0) {
   sarun ("-t < data/nice/spf2", \&patterns_run_cb);
   ok_all_patterns();
-} else {
-  ok(1);        # skip the tests
-  ok(1);
-}
 
 
-# 15-16: Trusted+Internal networks contain first header.
-
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.157
-internal_networks 65.214.43.157
-always_trust_envelope_sender 1
-");
+  # 13-14: Internal networks contain first header.
+  #	   Trusted networks contain some other IP.
+  #        jm: commented; this is now an error condition.
+
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 1.2.3.4
+    internal_networks 65.214.43.157
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%patterns = (
+  %patterns = (
+    q{ SPF_HELO_NEUTRAL }, 'helo_neutral',
+    q{ SPF_NEUTRAL }, 'neutral',
+  );
+
+  if (0) {
+    sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+    ok_all_patterns();
+  } else {
+    ok(1);        # skip the tests
+    ok(1);
+  }
+
+
+  # 15-16: Trusted+Internal networks contain first header.
+
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.157
+    internal_networks 65.214.43.157
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
+
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 17-18: Trusted networks contain first and second header.
-#	 Internal networks contain first header.
+  # 17-18: Trusted networks contain first and second header.
+  #	   Internal networks contain first header.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.157 64.142.3.173
-internal_networks 65.214.43.157
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.157 64.142.3.173
+    internal_networks 65.214.43.157
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%patterns = (
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 19-26: Trusted networks contain first and second header.
-#	 Internal networks contain first and second header.
+  # 19-26: Trusted networks contain first and second header.
+  #	   Internal networks contain first and second header.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.157 64.142.3.173
-internal_networks 65.214.43.157 64.142.3.173
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.157 64.142.3.173
+    internal_networks 65.214.43.157 64.142.3.173
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = (
+  %anti_patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_HELO_FAIL }, 'helo_fail',
     q{ SPF_HELO_SOFTFAIL }, 'helo_softfail',
@@ -195,114 +216,179 @@
     q{ SPF_FAIL }, 'fail',
     q{ SPF_SOFTFAIL }, 'softfail',
     q{ SPF_NEUTRAL }, 'neutral',
-);
-%patterns = ();
+  );
+  %patterns = ();
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 27-28: Trusted networks contain first header.
-#	 Internal networks contain first and second header.
+  # 27-28: Trusted networks contain first header.
+  #	   Internal networks contain first and second header.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.157
-internal_networks 65.214.43.157 64.142.3.173
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.157
+    internal_networks 65.214.43.157 64.142.3.173
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = ();
-%patterns = (
+  %anti_patterns = ();
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf2", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf2", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 29-30: Trusted networks contain top 5 headers.
-#	 Internal networks contain first header.
+  # 29-30: Trusted networks contain top 5 headers.
+  #	   Internal networks contain first header.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
-internal_networks 65.214.43.158
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
+    internal_networks 65.214.43.158
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = ();
-%patterns = (
+  %anti_patterns = ();
+  %patterns = (
     q{ SPF_HELO_PASS }, 'helo_pass',
     q{ SPF_PASS }, 'pass',
-);
+  );
 
-sarun ("-t < data/nice/spf3", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf3", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 31-32: Trusted networks contain top 5 headers.
-#	 Internal networks contain top 2 headers.
+  # 31-32: Trusted networks contain top 5 headers.
+  #	   Internal networks contain top 2 headers.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
-internal_networks 65.214.43.158 64.142.3.173
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
+    internal_networks 65.214.43.158 64.142.3.173
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = ();
-%patterns = (
+  %anti_patterns = ();
+  %patterns = (
     q{ SPF_HELO_FAIL }, 'helo_fail',
     q{ SPF_FAIL }, 'fail',
-);
+  );
 
-sarun ("-t < data/nice/spf3", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf3", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 33-34: Trusted networks contain top 5 headers.
-#	 Internal networks contain top 3 headers.
+  # 33-34: Trusted networks contain top 5 headers.
+  #	   Internal networks contain top 3 headers.
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
-internal_networks 65.214.43.158 64.142.3.173 65.214.43.155
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
+    internal_networks 65.214.43.158 64.142.3.173 65.214.43.155
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = ();
-%patterns = (
+  %anti_patterns = ();
+  %patterns = (
     q{ SPF_HELO_SOFTFAIL }, 'helo_softfail',
     q{ SPF_SOFTFAIL }, 'softfail',
-);
+  );
 
-sarun ("-t < data/nice/spf3", \&patterns_run_cb);
-ok_all_patterns();
+  sarun ("-t < data/nice/spf3", \&patterns_run_cb);
+  ok_all_patterns();
 
 
-# 35-36: Trusted networks contain top 5 headers.
-#	 Internal networks contain top 4 headers.	
+  # 35-36: Trusted networks contain top 5 headers.
+  #	   Internal networks contain top 4 headers.	
 
-tstprefs("
-clear_trusted_networks
-clear_internal_networks
-trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
-internal_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156
-always_trust_envelope_sender 1
-");
+  tstprefs("
+    clear_trusted_networks
+    clear_internal_networks
+    trusted_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156 65.214.43.157
+    internal_networks 65.214.43.158 64.142.3.173 65.214.43.155 65.214.43.156
+    always_trust_envelope_sender 1
+    $disable_an_spf_module
+  ");
 
-%anti_patterns = ();
-%patterns = (
+  %anti_patterns = ();
+  %patterns = (
     q{ SPF_HELO_NEUTRAL }, 'helo_neutral',
     q{ SPF_NEUTRAL }, 'neutral',
+  );
+
+  sarun ("-t < data/nice/spf3", \&patterns_run_cb);
+  ok_all_patterns();
+
+
+  # 37-40: same as test 1-2 with some spf whitelisting added
+
+  tstprefs("
+    whitelist_from_spf newsalerts-noreply\@dnsbltest.spamassassin.org
+    def_whitelist_from_spf *\@dnsbltest.spamassassin.org
+    $disable_an_spf_module
+  ");
+
+  %patterns = (
+    q{ SPF_HELO_PASS }, 'helo_pass',
+    q{ SPF_PASS }, 'pass',
+    q{ USER_IN_SPF_WHITELIST }, 'spf_whitelist',
+    q{ USER_IN_DEF_SPF_WL }, 'default_spf_whitelist',
+  );
+
+  sarun ("-t < data/nice/spf1", \&patterns_run_cb);
+  ok_all_patterns();
+
+
+  # 41-44: same as test 1-2 with some spf whitelist entires that don't match
+
+  tstprefs("
+    whitelist_from_spf *\@example.com
+    def_whitelist_from_spf nothere\@dnsbltest.spamassassin.org
+    $disable_an_spf_module
+  ");
+
+  %patterns = (
+    q{ SPF_HELO_PASS }, 'helo_pass',
+    q{ SPF_PASS }, 'pass',
+  );
+
+  %anti_patterns = (
+    q{ USER_IN_SPF_WHITELIST }, 'spf_whitelist',
+    q{ USER_IN_DEF_SPF_WL }, 'default_spf_whitelist',
+  );
+
+  sarun ("-t < data/nice/spf1", \&patterns_run_cb);
+  ok_all_patterns();
+
+  # clear these out before we loop
+  %anti_patterns = ();
+  %patterns = ();
+
+} # for each SPF module
+
+
+# test to see if the plugin will select an SPF module on its own
+
+tstprefs("");
+
+%patterns = (
+    q{ SPF_HELO_PASS }, 'helo_pass',
+    q{ SPF_PASS }, 'pass',
 );
 
-sarun ("-t < data/nice/spf3", \&patterns_run_cb);
+sarun ("-t < data/nice/spf1", \&patterns_run_cb);
 ok_all_patterns();