You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by he...@apache.org on 2022/05/13 06:06:33 UTC
svn commit: r1900849 - in /spamassassin/trunk: lib/Mail/SpamAssassin/ lib/Mail/SpamAssassin/Plugin/ t/
Author: hege
Date: Fri May 13 06:06:33 2022
New Revision: 1900849
URL: http://svn.apache.org/viewvc?rev=1900849&view=rev
Log:
- Bug 7987
- fix body rules considered unrun when using sa-compile
- fix check_rbl_sub rules considered unrun and other DNSEval cleanups
- improve rule_pending/rule_ready/got_hit() logic
- rename $pms->get_pending_lookups to get_async_pending_rules
- other minor async cleanups
- test and documentation improvements
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
spamassassin/trunk/t/basic_meta2.t
spamassassin/trunk/t/basic_meta_net.t
spamassassin/trunk/t/dnsbl.t
spamassassin/trunk/t/sa_compile.t
spamassassin/trunk/t/uribl.t
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm Fri May 13 06:06:33 2022
@@ -73,7 +73,7 @@ sub new {
queries_started => 0,
queries_completed => 0,
pending_lookups => { },
- pending_rules => { },
+ pending_rules => { }, # maintain pending rules list for meta evaluation
rules_for_key => { }, # record all rules used by a key for logging
timing_by_query => { },
all_lookups => { }, # keyed by "rr_type/domain"
@@ -181,8 +181,17 @@ Hash of options. Only supported and requ
=cut
+sub start_queue {
+ my($self) = @_;
+
+ $self->{wait_queue} = 1;
+}
+
sub launch_queue {
my($self) = @_;
+
+ delete $self->{wait_queue};
+
if ($self->{bgsend_queue}) {
dbg("async: launching queued lookups");
foreach (@{$self->{bgsend_queue}}) {
@@ -199,14 +208,17 @@ sub bgsend_and_start_lookup {
return if $self->{main}->{resolver}->{no_resolver};
# Waiting for priority -100 to launch?
- if ($self->{wait_launch}) {
+ if ($self->{wait_queue}) {
push @{$self->{bgsend_queue}}, [@_];
- dbg("async: dns priority not reached, queueing lookup: $domain/$type");
+ dbg("async: DNS priority not reached, queueing lookup: $domain/$type");
return $ent;
}
- if (!defined $ent->{rulename}) {
- info("async: bgsend_and_start_lookup called without rulename: $domain/$type");
+ if (!defined $ent->{rulename} && !$self->{rulename_warned}++) {
+ my($package, $filename, $line) = caller;
+ warn "async: bgsend_and_start_lookup called without rulename, ".
+ "from $package ($filename) line $line. You are likely using ".
+ "a plugin that is not compatible with SpamAssasin 4.0.0.";
}
$domain =~ s/\.+\z//s; # strip trailing dots, these sometimes still sneak in
@@ -316,9 +328,10 @@ sub bgsend_and_start_lookup {
}
}
if ($blocked) {
- dbg("async: blocked by %s: %s", $blocked_by, $dnskey);
+ dbg("async: blocked by %s: %s, rules: %s", $blocked_by, $dnskey,
+ join(", ", @rulenames));
} else {
- dbg("async: launching %s", $dnskey);
+ dbg("async: launching %s, rules: %s", $dnskey, join(", ", @rulenames));
$id = $self->{main}->{resolver}->bgsend($domain, $type, $class, sub {
my($pkt, $pkt_id, $timestamp) = @_;
# this callback sub is called from DnsResolver::poll_responses()
@@ -376,9 +389,12 @@ DIRECT USE DEPRECATED since 4.0.0, pleas
sub start_lookup {
my $self = shift;
- warn "async: deprecated start_lookup called, please use bgsend_and_start_lookup\n"
- if !$self->{start_lookup_warned};
- $self->{start_lookup_warned} = 1;
+ if (!$self->{start_lookup_warned}++) {
+ my($package, $filename, $line) = caller;
+ warn "async: deprecated start_lookup called, ".
+ "from $package ($filename) line $line. You are likely using ".
+ "a plugin that is not compatible with SpamAssasin 4.0.0.";
+ }
return if $self->{main}->{resolver}->{no_resolver};
$self->_start_lookup(@_);
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm Fri May 13 06:06:33 2022
@@ -110,6 +110,8 @@ sub do_rbl_lookup {
dbg("dns: launching rule %s, set %s, type %s, %s", $rule, $set, $type,
defined $subtest ? "subtest $subtest" : 'no subtest');
+ $self->rule_pending($rule); # mark async
+
my $ent = {
rulename => $rule,
type => "DNSBL",
@@ -126,6 +128,8 @@ sub do_rbl_lookup {
sub do_dns_lookup {
my ($self, $rule, $type, $host) = @_;
+ $self->rule_pending($rule); # mark async
+
my $ent = {
rulename => $rule,
type => "DNSBL",
@@ -222,6 +226,20 @@ sub process_dnsbl_result {
my $question = ($pkt->question)[0];
return if !$question;
+ my $rulename = $ent->{rulename};
+
+ # Mark rule ready for meta rules, but only if this was the last lookup
+ # pending, rules can have many lookups launched for different IPs
+ if (!$self->get_async_pending_rules($rulename)) {
+ $self->rule_ready($rulename);
+ # Mark depending check_rbl_sub rules too
+ if (exists $self->{rbl_subs}{$ent->{set}}) {
+ foreach (@{$self->{rbl_subs}{$ent->{set}}}) {
+ $self->rule_ready($_->[1]);
+ }
+ }
+ }
+
# DNSBL tests are here
foreach my $answer ($pkt->answer) {
next if !$answer;
@@ -256,17 +274,18 @@ sub process_dnsbl_result {
# check_rbl tests
if (defined $ent->{subtest}) {
if ($self->check_subtest($rdatastr, $ent->{subtest})) {
- $self->dnsbl_hit($ent->{rulename}, $question, $answer);
+ $self->dnsbl_hit($rulename, $question, $answer);
}
} else {
- $self->dnsbl_hit($ent->{rulename}, $question, $answer);
+ $self->dnsbl_hit($rulename, $question, $answer);
}
# check_rbl_sub tests
- if (defined $self->{rbl_subs}{$ent->{set}}) {
+ if (exists $self->{rbl_subs}{$ent->{set}}) {
$self->process_dnsbl_set($ent->{set}, $question, $answer, $rdatastr);
}
}
+
return 1;
}
@@ -685,9 +704,9 @@ sub register_async_rule_finish {}
sub mark_all_async_rules_complete {}
sub is_rule_complete {}
-# Return number of pending lookups for a rule,
+# Return number of pending DNS lookups for a rule,
# or list all of rules still pending
-sub get_pending_lookups {
+sub get_async_pending_rules {
my ($self, $rule) = @_;
if (defined $rule) {
return 0 if !exists $self->{async}->{pending_rules}{$rule};
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Fri May 13 06:06:33 2022
@@ -3104,6 +3104,9 @@ sub got_hit {
$score = $params{defscore} if !defined $score;
}
+ # Make sure rule is marked ready in tests_pending also
+ $self->{tests_pending}->{$rule} = 0;
+
# adding a hit does nothing if we don't have a score -- we probably
# shouldn't have run it in the first place
if (!$score) {
@@ -3170,20 +3173,25 @@ sub got_hit {
=item $status->rule_pending ($rulename)
-Register a pending rule. Must be called from rules eval-function, if the
-result can arrive later than when exiting the function (async lookups).
-This is not required if eval uses $status->{async}->bgsend_and_start_lookup(),
-as it automatically registers a pending rule and also marks it ready when the
-result callback is run.
-
-$status->rule_ready($rulename) or $status->got_hit(...) must be called when
-the result has arrived. If these are not used, it can break dynamic meta rule
-evaluation. This does not apply when bgsend_and_start_lookup() was used, as
-mentioned above.
+Register a pending rule. Must be called if the rules result is evaluated
+later than when exiting it's eval-function (e.g. async lookups). This is
+required for meta rules that depend on this rule to evaluate correctly.
+
+It is not strictly required when $status->{async}->bgsend_and_start_lookup()
+is used, as that automatically tracks pending DNS rules and also marks them
+ready when the result callback is run. But when a plugin has dummy stub
+eval-function (examples: check_uridnsbl, check_rbl_sub), then using
+rule_pending() and rule_ready() is necessary. Otherwise in some cases the
+eval-function could be run before any lookups are launched, and the rule
+would be marked ready too early.
+
+$status->rule_ready($rulename) or $status->got_hit($rulename, ...) must be
+called when the result has arrived. If these are not used, then any meta
+rules depending on the rule might not evaluate at all.
=cut
-# === Bug 7735 notes ===
+# === Bug 7735 technical notes ===
#
# Rule readiness status for dynamic meta rules evaluation (Plugins / Check /
# do_meta_tests) is always tracked by $pms->{tests_already_hit}->{$rule}.
@@ -3191,9 +3199,12 @@ mentioned above.
#
# The public rule_pending() / rule_ready() function pair is meant to be used
# for any async rules, and they additionally track rule status in
-# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. So any
-# rule_pending() call must always be accompanied later by rule_ready() or
-# got_hit(). These are not required when bgsend_and_start_lookup() is used.
+# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. Any
+# rule_pending() call _must_ be accompanied later by rule_ready() or
+# got_hit(). But as bgsend_and_start_lookup() automatically tracks rules
+# too, most of legacy plugins not using these should work correctly - only
+# in some corner cases where a dummy stub eval-function is called early by
+# accident (maybe due to adjusted rule priority) things might break.
#
# Any rule regardless of what it is must always be marked ready via
# $pms->{tests_already_hit} for meta evaluation to work, even if no
@@ -3201,38 +3212,43 @@ mentioned above.
# use $pms->{tests_already_hit} directly for marking, as is done for example
# by Check.pm for all basic body/header/etc rules (look for
# $hitsptr->{q{'.$rulename.'}} ||= 0). As such it's also always safe to
-# call rule_ready() without rule_pending().
+# call rule_ready() without rule_pending(), it achieves the same.
sub rule_pending {
my ($self, $rule) = @_;
+ # Ignore if called after rule_pending(), rule_ready() or got_hit(), they
+ # are the only ones that make $self->{tests_pending}->{$rule} exist
+ return if exists $self->{tests_pending}->{$rule};
+
+ # It's possible that tests_already_hit == 0, if some stub eval call was
+ # run early. It's ok to clear this and pretend it was never run. We now
+ # rely exclusively on tests_pending and wait for rule_ready() or got_hit()
+ delete $self->{tests_already_hit}->{$rule} if !$self->{tests_already_hit}->{$rule};
$self->{tests_pending}->{$rule} = 1;
-
- if (exists $self->{tests_already_hit}->{$rule}) {
- # Only clear result if not hit
- if ($self->{tests_already_hit}->{$rule} == 0) {
- delete $self->{tests_already_hit}->{$rule};
- }
- }
}
=item $status->rule_ready ($rulename)
-Mark a previously marked $status->rule_pending() rule ready. Alternatively
-$status->got_hit() will also mark rule ready. If these are not used, it can
-break dynamic meta rule evaluation.
+Mark a rule ready, so it can be considered for meta rule evaluation. Any
+rule regardless of type must always be marked ready when it's finished,
+otherwise any meta rules that depend on it might not evaluate. If
+$status->rule_pending() was called, then a $stats->rule_ready() call must be
+done, alternatively $status->got_hit() will also mark rule ready.
=cut
sub rule_ready {
my ($self, $rule) = @_;
- if ($self->get_pending_lookups($rule)) {
- # Can't be ready if there are pending lookups, ignore for now.
+ if ($self->get_async_pending_rules($rule)) {
+ # Can't be ready if there are pending DNS lookups, ignore for now.
+ # Make sure tests_pending exist, should not accept rule_pending() anymore
+ $self->{tests_pending}->{$rule} ||= 0;
return;
}
- delete $self->{tests_pending}->{$rule};
+ $self->{tests_pending}->{$rule} = 0;
$self->{tests_already_hit}->{$rule} ||= 0;
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm Fri May 13 06:06:33 2022
@@ -433,13 +433,14 @@ sub launch_queries {
next;
}
dbg("askdns: launching query ($rulename): $query");
- $pms->{async}->bgsend_and_start_lookup(
+ my $ret = $pms->{async}->bgsend_and_start_lookup(
$query, $arule->{q_type}, undef,
{ rulename => $rulename, type => 'AskDNS' },
sub { my ($ent,$pkt) = @_;
$self->process_response_packet($pms, $ent, $pkt, $rulename) },
master_deadline => $pms->{master_deadline}
);
+ $pms->rule_ready($rulename) if !$ret; # mark ready if nothing launched
}
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Fri May 13 06:06:33 2022
@@ -54,8 +54,8 @@ sub check_main {
my $pms = $args->{permsgstatus};
$would_log_rules_all = would_log('dbg', 'rules-all') == 2;
- # Make AsyncLoop wait permission for launching queries
- $pms->{async}->{wait_launch} = 1;
+ # Make AsyncLoop wait launch_queue() for launching queries
+ $pms->{async}->start_queue();
my $suppl_attrib = $pms->{msg}->{suppl_attrib};
if (ref $suppl_attrib && ref $suppl_attrib->{rule_hits}) {
@@ -129,7 +129,6 @@ sub check_main {
# unneeded DNS queries.
if ($do_dns && !$rbls_running && $priority >= -100) {
$rbls_running = 1;
- $pms->{async}->{wait_launch} = 0; # permission granted
$pms->{async}->launch_queue(); # check if something was queued
$self->run_rbl_eval_tests($pms);
$self->{main}->call_plugins ("check_dnsbl", { permsgstatus => $pms });
@@ -289,8 +288,8 @@ sub do_meta_tests {
my $h = $pms->{tests_already_hit};
my $retry;
- # Get pending async rule list
- my %pl = map { $_ => 1 } $pms->get_pending_lookups();
+ # Get pending DNS async rule list
+ my %pl = map { $_ => 1 } $pms->get_async_pending_rules();
RULE:
foreach my $rulename (keys %$mp) {
@@ -323,6 +322,7 @@ sub finish_meta_tests {
return if $self->{am_compiling}; # nothing to compile here
my $mp = $pms->{meta_pending};
+ my $tp = $pms->{tests_pending};
my $md = $pms->{conf}->{meta_dependencies};
my $mt = $pms->{conf}->{meta_tests};
my $h = $pms->{tests_already_hit};
@@ -334,7 +334,7 @@ RULE:
my %unrun;
# Meta is not ready if some dependency has not run yet
foreach my $deprule (@{$md->{$rulename}||[]}) {
- if (!exists $h->{$deprule}) {
+ if (!exists $h->{$deprule} || $tp->{$deprule}) {
# Record all unrun deps for second meta evaluation
$unrun{$deprule} = 1;
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm Fri May 13 06:06:33 2022
@@ -150,13 +150,14 @@ sub set_config {
sub finish_parsing_start {
my ($self, $opts) = @_;
+ my $conf = $opts->{conf};
# Adjust priority -100 to launch early
# Find rulenames from eval_to_rule mappings
foreach my $evalfunc (@{$self->{'evalrules'}}) {
- foreach (@{$opts->{conf}->{eval_to_rule}->{$evalfunc}||[]}) {
+ foreach (@{$conf->{eval_to_rule}->{$evalfunc}||[]}) {
dbg("dnseval: adjusting rule $_ priority to -100");
- $opts->{conf}->{priority}->{$_} = -100;
+ $conf->{priority}->{$_} = -100;
}
}
}
@@ -165,34 +166,33 @@ sub finish_parsing_start {
# directly as part of PMS
sub check_start {
my ($self, $opts) = @_;
+ my $pms = $opts->{permsgstatus};
foreach(@{$self->{'evalrules'}}) {
- $opts->{'permsgstatus'}->register_plugin_eval_glue($_);
+ $pms->register_plugin_eval_glue($_);
}
# Initialize check_rbl_sub tests
- $self->init_rbl_subs($opts->{'permsgstatus'});
+ $self->_init_rbl_subs($pms);
}
-sub init_rbl_subs {
+sub _init_rbl_subs {
my ($self, $pms) = @_;
-
- return if $pms->{rbl_subs};
+ my $conf = $pms->{conf};
# Very hacky stuff and direct rbl_evals usage for now, TODO rewrite everything
- foreach my $rule (@{$pms->{conf}->{eval_to_rule}->{check_rbl_sub}}) {
- next if !exists $pms->{conf}->{rbl_evals}->{$rule};
- next if !$pms->{conf}->{scores}->{$rule};
+ foreach my $rule (@{$conf->{eval_to_rule}->{check_rbl_sub}||[]}) {
+ next if !exists $conf->{rbl_evals}->{$rule};
+ next if !$conf->{scores}->{$rule};
# rbl_evals is [$function,[@args]]
- my $args = $pms->{conf}->{rbl_evals}->{$rule}->[1];
- my $set = $args->[0];
- my $subtest = $args->[1];
+ my $args = $conf->{rbl_evals}->{$rule}->[1];
+ my ($set, $subtest) = @$args;
if (!defined $subtest) {
warn("dnseval: missing subtest for rule $rule\n");
next;
}
if ($subtest =~ /^sb:/) {
- info("dnseval: ignored $rule, SenderBase rules are deprecated");
+ warn("dnseval: ignored $rule, SenderBase rules are deprecated\n");
next;
}
# Compile as regex if not pure ip/bitmask (same check in process_dnsbl_result)
@@ -206,6 +206,7 @@ sub init_rbl_subs {
}
dbg("dnseval: initialize check_rbl_sub for rule $rule, set $set, subtest $subtest");
push @{$pms->{rbl_subs}{$set}}, [$subtest, $rule];
+ $pms->rule_pending($rule); # mark async, rule_ready() done in Dns/process_dnsbl_result
}
}
@@ -283,7 +284,7 @@ sub check_rbl_accreditor {
$self->message_accreditor_tag($pms);
}
if ($pms->{accreditor_tag}->{$accreditor}) {
- $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
+ $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
}
return 0;
}
@@ -324,7 +325,7 @@ sub message_accreditor_tag {
$pms->{accreditor_tag} = \%acctags;
}
-sub check_rbl_backend {
+sub _check_rbl_backend {
my ($self, $pms, $rule, $set, $rbl_server, $type, $subtest) = @_;
return if !exists $pms->{dnseval_ips}; # no untrusted ips
@@ -406,14 +407,12 @@ sub check_rbl_backend {
return 0;
}
- $pms->rule_pending($rule); # mark async
-
dbg("dnseval: only inspecting the following IPs: ".join(", ", @ips));
foreach my $ip (@ips) {
- my $revip = reverse_ip_address($ip);
- $pms->do_rbl_lookup($rule, $set, $type,
- $revip.'.'.$rbl_server, $subtest) if defined $revip;
+ if (defined(my $revip = reverse_ip_address($ip))) {
+ $pms->do_rbl_lookup($rule, $set, $type, $revip.'.'.$rbl_server, $subtest)
+ }
}
# note that results are not handled here, hits are handled directly
@@ -427,7 +426,7 @@ sub check_rbl {
return 0 if $self->{main}->{conf}->{skip_rbl_checks};
return 0 if !$pms->is_dns_available();
- $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
+ $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
}
sub check_rbl_txt {
@@ -436,12 +435,12 @@ sub check_rbl_txt {
return 0 if $self->{main}->{conf}->{skip_rbl_checks};
return 0 if !$pms->is_dns_available();
- $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest);
+ $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest);
}
sub check_rbl_sub {
my ($self, $pms, $rule, $set, $subtest) = @_;
- # just a dummy, check_start / init_rbl_subs handles the subs
+ # just a dummy, _init_rbl_subs/do_rbl_lookup handles the subs
$pms->rule_pending($rule); # mark async
return 0;
}
@@ -454,8 +453,6 @@ sub check_rbl_from_host {
return 0 if $self->{main}->{conf}->{skip_rbl_checks};
return 0 if !$pms->is_dns_available();
- $pms->rule_pending($rule); # mark async
-
$self->_check_rbl_addresses($pms, $rule, $set, $rbl_server,
$subtest, $pms->all_from_addrs());
}
@@ -551,10 +548,11 @@ sub check_rbl_ns_from {
dbg("dnseval: checking NS for host $domain");
- my $key = "NS:" . $domain;
+ $pms->rule_pending($rule); # mark async
+
my $obj = { dom => $domain, rule => $rule, set => $set, rbl_server => $rbl_server, subtest => $subtest };
my $ent = {
- rulename => $rule, key => $key, zone => $domain, obj => $obj, type => "URI-NS",
+ rulename => $rule, zone => $domain, obj => $obj, type => "URI-NS",
};
# dig $dom ns
$ent = $pms->{async}->bgsend_and_start_lookup(
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm Fri May 13 06:06:33 2022
@@ -66,6 +66,21 @@ sub check_start {
}
}
+sub check_cleanup {
+ my ($self, $params) = @_;
+ my $pms = $params->{permsgstatus};
+ my $hitsptr = $pms->{tests_already_hit};
+ my $scoresptr = $pms->{conf}->{scores};
+
+ # Force all body rules ready for meta rules. Need to do it here in
+ # cleanup, because the body is scanned per line instead of per rule
+ if ($pms->{conf}->{skip_body_rules}) {
+ foreach (keys %{$pms->{conf}->{skip_body_rules}}) {
+ $hitsptr->{$_} ||= 0 if $scoresptr->{$_};
+ }
+ }
+}
+
###########################################################################
1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm Fri May 13 06:06:33 2022
@@ -117,6 +117,11 @@ sub check_rules_at_priority {
$self->{one_line_body}->check_rules_at_priority($params);
}
+sub check_cleanup {
+ my ($self, $params) = @_;
+ $self->{one_line_body}->check_cleanup($params);
+}
+
###########################################################################
sub run_body_fast_scan {
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm Fri May 13 06:06:33 2022
@@ -197,6 +197,11 @@ sub check_rules_at_priority {
$self->{one_line_body}->check_rules_at_priority($params);
}
+sub check_cleanup {
+ my ($self, $params) = @_;
+ $self->{one_line_body}->check_cleanup($params);
+}
+
###########################################################################
sub run_body_fast_scan {
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Fri May 13 06:06:33 2022
@@ -378,6 +378,8 @@ sub check_dnsbl {
foreach my $rulename (keys %{$conf->{uridnsbls}}) {
next if !$conf->{scores}->{$rulename};
+ $pms->rule_pending($rulename); # mark async
+
my $rulecf = $conf->{uridnsbls}->{$rulename};
my %tfl = map { ($_,1) } split(/\s+/, $conf->{tflags}->{$rulename}||'');
Modified: spamassassin/trunk/t/basic_meta2.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta2.t?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/t/basic_meta2.t (original)
+++ spamassassin/trunk/t/basic_meta2.t Fri May 13 06:06:33 2022
@@ -5,7 +5,7 @@ use lib 't';
use SATest; sa_t_init("basic_meta2");
use Test::More;
-plan tests => 15;
+plan tests => 20;
# ---------------------------------------------------------------------------
@@ -19,6 +19,11 @@ plan tests => 15;
q{ TEST_META_7 } => '',
q{ TEST_META_A } => '',
q{ TEST_META_B } => '',
+ q{ TEST_META_C } => '',
+ q{ TEST_META_D } => '',
+ q{ TEST_META_E } => '',
+ q{ TEST_META_F } => '',
+ q{ TEST_META_G } => '',
);
%anti_patterns = (
@@ -80,6 +85,21 @@ tstlocalrules (qq{
# local_tests_only
meta TEST_META_B NONEXISTINGRULE || local_tests_only
+ # complex metas with different priorities
+ body __BAR_5 /a/
+ priority __BAR_5 -1000
+ body __BAR_6 /b/
+ priority __BAR_6 0
+ body __BAR_7 /c/
+ priority __BAR_7 1000
+ meta TEST_META_C __BAR_5 && __BAR_6 && __BAR_7
+ meta TEST_META_D __BAR_5 && __BAR_6 && TEST_META_C
+ priority TEST_META_D -2000
+ meta TEST_META_E __BAR_6 && __BAR_7 && TEST_META_D
+ meta TEST_META_F __BAR_5 && __BAR_7 && TEST_META_E
+ priority TEST_META_F 2000
+ meta TEST_META_G TEST_META_C && TEST_META_D && TEST_META_E && TEST_META_F
+
});
sarun ("-L -t < data/nice/001 2>&1", \&patterns_run_cb);
Modified: spamassassin/trunk/t/basic_meta_net.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta_net.t?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/t/basic_meta_net.t (original)
+++ spamassassin/trunk/t/basic_meta_net.t Fri May 13 06:06:33 2022
@@ -8,79 +8,87 @@ use Test::More;
plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
-plan tests => 20;
+plan tests => 32;
# ---------------------------------------------------------------------------
%patterns = (
- q{ X_META_POS4 } => '',
+ q{ 1.0 X_LOCAL_TESTS } => '',
);
%anti_patterns = (
- q{ X_URIBL_A } => '',
- q{ X_ASKDNS } => '',
- q{ X_META_POS1 } => '',
- q{ X_META_POS2 } => '',
- q{ X_META_POS3 } => '',
- q{ X_META_NEG1 } => '',
- q{ X_META_NEG2 } => '',
- q{ X_META_NEG3 } => '',
- q{ X_META_NEG4 } => '',
+ q{ 1.0 X_URIBL_A } => '',
+ q{ 1.0 X_ASKDNS } => '',
+ q{ 1.0 X_DNSBL_TEST } => '',
+ q{ 1.0 X_DNSBL_SUB } => '',
+ q{ 1.0 X_META_POS1 } => '',
+ q{ 1.0 X_META_POS2 } => '',
+ q{ 1.0 X_META_POS3 } => '',
+ q{ 1.0 X_META_POS4 } => '',
+ q{ 1.0 X_META_POS5 } => '',
+ q{ 1.0 X_META_NEG1 } => '',
+ q{ 1.0 X_META_NEG2 } => '',
+ q{ 1.0 X_META_NEG3 } => '',
+ q{ 1.0 X_META_NEG4 } => '',
+ q{ 1.0 X_META_NEG5 } => '',
+ q{ 1.0 X_LOCAL_NEG } => '',
);
-#
-# Nothing should hit with a failed lookup
-#
-
-tstlocalrules (qq{
- # Force DNS queries to fail/timeout
- rbl_timeout 2 1
- dns_server 240.0.0.240
-
+my $common_rules = q{
urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2
body X_URIBL_A eval:check_uridnsbl('X_URIBL_A')
tflags X_URIBL_A net
askdns X_ASKDNS spamassassin.org TXT /./
+ header X_DNSBL_TEST eval:check_rbl('test', 'dnsbltest.spamassassin.org.')
+ tflags X_DNSBL_TEST net
+
+ header X_DNSBL_SUB eval:check_rbl_sub('test', '2')
+ tflags X_DNSBL_SUB net
+
meta X_META_POS1 X_URIBL_A
meta X_META_POS2 X_ASKDNS
- meta X_META_POS3 X_URIBL_A || X_ASKDNS
+ meta X_META_POS3 X_DNSBL_TEST
+ meta X_META_POS4 X_DNSBL_SUB
+ meta X_META_POS5 X_URIBL_A || X_ASKDNS || X_DNSBL_TEST || X_DNSBL_SUB
meta X_META_NEG1 !X_URIBL_A
meta X_META_NEG2 !X_ASKDNS
- meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS
+ meta X_META_NEG3 !X_DNSBL_TEST
+ meta X_META_NEG4 !X_DNSBL_SUB
+ meta X_META_NEG5 !X_URIBL_A || !X_ASKDNS || !X_DNSBL_TEST || !X_DNSBL_SUB
+};
+
+#
+# Nothing should hit with a timed out lookup
+#
+
+tstlocalrules (qq{
+ # Force DNS queries to fail/timeout
+ rbl_timeout 2 1
+ dns_server 240.0.0.240
+
+ $common_rules
# local_tests_only
- meta X_META_NEG4 local_tests_only
- meta X_META_POS4 !local_tests_only
+ meta X_LOCAL_TESTS !local_tests_only
+ meta X_LOCAL_NEG local_tests_only
});
-sarun ("-t < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb);
+sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
ok_all_patterns();
#
-# Local only, nothing should hit as nothing is queried
+# Local tests only, nothing should hit as nothing is queried
#
tstlocalrules (qq{
- urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2
- body X_URIBL_A eval:check_uridnsbl('X_URIBL_A')
- tflags X_URIBL_A net
-
- askdns X_ASKDNS spamassassin.org TXT /./
-
- meta X_META_POS1 X_URIBL_A
- meta X_META_POS2 X_ASKDNS
- meta X_META_POS3 X_URIBL_A || X_ASKDNS
-
- meta X_META_NEG1 !X_URIBL_A
- meta X_META_NEG2 !X_ASKDNS
- meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS
+ $common_rules
# local_tests_only
- meta X_META_POS4 local_tests_only
- meta X_META_NEG4 !local_tests_only
+ meta X_LOCAL_TESTS local_tests_only
+ meta X_LOCAL_NEG !local_tests_only
});
sarun ("-t -L < data/spam/dnsbl.eml", \&patterns_run_cb);
Modified: spamassassin/trunk/t/dnsbl.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/dnsbl.t?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/t/dnsbl.t (original)
+++ spamassassin/trunk/t/dnsbl.t Fri May 13 06:06:33 2022
@@ -6,7 +6,10 @@ use SATest; sa_t_init("dnsbl");
use Test::More;
plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
-plan tests => 21;
+
+# run many times to catch some random natured failures
+my $iterations = 5;
+plan tests => 22 * $iterations;
# ---------------------------------------------------------------------------
# bind configuration currently used to support this test
@@ -71,6 +74,7 @@ EOF
q{'1.0 DNSBL_TXT_MISS'} => '',
q{'1.0 DNSBL_TEST_WHITELIST_MISS'} => '',
q{'14.35.17.212.untrusted.dnsbltest.spamassassin.org'} => '',
+ q{/rules-all: unrun dependencies [^\n]+ (?:__|META_)?DNSBL_/} => '',
);
tstprefs("
@@ -148,6 +152,9 @@ priority DNSBL_TEST_RELAY 2000
");
-sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
-ok_all_patterns();
+for (1 .. $iterations) {
+ # rules-all debug needed for unrun check
+ sarun ("-t -D rules-all < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb);
+ ok_all_patterns();
+}
Modified: spamassassin/trunk/t/sa_compile.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/sa_compile.t?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/t/sa_compile.t (original)
+++ spamassassin/trunk/t/sa_compile.t Fri May 13 06:06:33 2022
@@ -54,35 +54,36 @@ body FOO5 /éèæ©/
body FOO6 /éè(?:xyz|æ©)/
body FOO7 /\xe9\x87\x91\xe8\x9e\x8d\xe6\xa9\x9f/
body FOO8 /.\x87(?:\x91|\x00)[\xe8\x00]\x9e\x8d\xe6\xa9\x9f/
+# Test that meta rules work for sa-compiled body rules
+# (loosely related to Bug 7987)
+meta META1 FOO1 && FOO2 && FOO3 && FOO4
+meta META2 FOO5 && FOO6 && FOO7 && FOO8
');
# ensure we don't use compiled rules
untaint_system("rm -rf $instdir/var/spamassassin/compiled");
%patterns = (
- '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO',
+ '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '',
);
%anti_patterns = (
'/ zoom: able to use /', '',
);
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
+
%patterns = (
- '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO',
+ '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '',
);
%anti_patterns = (
'/ zoom: able to use /', '',
);
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
# -------------------------------------------------------------------
@@ -92,27 +93,24 @@ $scr = "$instdir/$temp_binpath/spamassas
$scr_localrules_args = $scr_cf_args = ""; # use the default rules dir, from our "install"
%patterns = (
- q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use',
- '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO',
+ q{ zoom: able to use 5/5 'body_0' compiled rules }, '',
+ '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '',
);
%anti_patterns = ();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
+
%patterns = (
- q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use',
- '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO',
+ q{ zoom: able to use 5/5 'body_0' compiled rules }, '',
+ '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '',
);
%anti_patterns = ();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
-ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
+ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
ok_all_patterns();
-clear_pattern_counters();
# -------------------------------------------------------------------
Modified: spamassassin/trunk/t/uribl.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/uribl.t?rev=1900849&r1=1900848&r2=1900849&view=diff
==============================================================================
--- spamassassin/trunk/t/uribl.t (original)
+++ spamassassin/trunk/t/uribl.t Fri May 13 06:06:33 2022
@@ -6,7 +6,10 @@ use SATest; sa_t_init("uribl");
use Test::More;
plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
-plan tests => 10;
+
+# run many times to catch some random natured failures
+my $iterations = 5;
+plan tests => 10 * $iterations;
# ---------------------------------------------------------------------------
@@ -68,6 +71,8 @@ tstlocalrules(q{
});
-ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
-ok_all_patterns();
+for (1 .. $iterations) {
+ ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
+ ok_all_patterns();
+}
Attention trunk & sa-compile users
Posted by Henrik K <he...@hege.li>.
I suggest updating your trunk version if using sa-compile, many metas
depending on body rules might not have run at all..
Also check_rbl_sub rules not going occasionally missing will help..
On Fri, May 13, 2022 at 06:06:33AM -0000, hege@apache.org wrote:
> Author: hege
> Date: Fri May 13 06:06:33 2022
> New Revision: 1900849
>
> URL: http://svn.apache.org/viewvc?rev=1900849&view=rev
> Log:
> - Bug 7987
> - fix body rules considered unrun when using sa-compile
> - fix check_rbl_sub rules considered unrun and other DNSEval cleanups
> - improve rule_pending/rule_ready/got_hit() logic
> - rename $pms->get_pending_lookups to get_async_pending_rules
> - other minor async cleanups
> - test and documentation improvements
>
> Modified:
> spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm
> spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
> spamassassin/trunk/t/basic_meta2.t
> spamassassin/trunk/t/basic_meta_net.t
> spamassassin/trunk/t/dnsbl.t
> spamassassin/trunk/t/sa_compile.t
> spamassassin/trunk/t/uribl.t
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm Fri May 13 06:06:33 2022
> @@ -73,7 +73,7 @@ sub new {
> queries_started => 0,
> queries_completed => 0,
> pending_lookups => { },
> - pending_rules => { },
> + pending_rules => { }, # maintain pending rules list for meta evaluation
> rules_for_key => { }, # record all rules used by a key for logging
> timing_by_query => { },
> all_lookups => { }, # keyed by "rr_type/domain"
> @@ -181,8 +181,17 @@ Hash of options. Only supported and requ
>
> =cut
>
> +sub start_queue {
> + my($self) = @_;
> +
> + $self->{wait_queue} = 1;
> +}
> +
> sub launch_queue {
> my($self) = @_;
> +
> + delete $self->{wait_queue};
> +
> if ($self->{bgsend_queue}) {
> dbg("async: launching queued lookups");
> foreach (@{$self->{bgsend_queue}}) {
> @@ -199,14 +208,17 @@ sub bgsend_and_start_lookup {
> return if $self->{main}->{resolver}->{no_resolver};
>
> # Waiting for priority -100 to launch?
> - if ($self->{wait_launch}) {
> + if ($self->{wait_queue}) {
> push @{$self->{bgsend_queue}}, [@_];
> - dbg("async: dns priority not reached, queueing lookup: $domain/$type");
> + dbg("async: DNS priority not reached, queueing lookup: $domain/$type");
> return $ent;
> }
>
> - if (!defined $ent->{rulename}) {
> - info("async: bgsend_and_start_lookup called without rulename: $domain/$type");
> + if (!defined $ent->{rulename} && !$self->{rulename_warned}++) {
> + my($package, $filename, $line) = caller;
> + warn "async: bgsend_and_start_lookup called without rulename, ".
> + "from $package ($filename) line $line. You are likely using ".
> + "a plugin that is not compatible with SpamAssasin 4.0.0.";
> }
>
> $domain =~ s/\.+\z//s; # strip trailing dots, these sometimes still sneak in
> @@ -316,9 +328,10 @@ sub bgsend_and_start_lookup {
> }
> }
> if ($blocked) {
> - dbg("async: blocked by %s: %s", $blocked_by, $dnskey);
> + dbg("async: blocked by %s: %s, rules: %s", $blocked_by, $dnskey,
> + join(", ", @rulenames));
> } else {
> - dbg("async: launching %s", $dnskey);
> + dbg("async: launching %s, rules: %s", $dnskey, join(", ", @rulenames));
> $id = $self->{main}->{resolver}->bgsend($domain, $type, $class, sub {
> my($pkt, $pkt_id, $timestamp) = @_;
> # this callback sub is called from DnsResolver::poll_responses()
> @@ -376,9 +389,12 @@ DIRECT USE DEPRECATED since 4.0.0, pleas
> sub start_lookup {
> my $self = shift;
>
> - warn "async: deprecated start_lookup called, please use bgsend_and_start_lookup\n"
> - if !$self->{start_lookup_warned};
> - $self->{start_lookup_warned} = 1;
> + if (!$self->{start_lookup_warned}++) {
> + my($package, $filename, $line) = caller;
> + warn "async: deprecated start_lookup called, ".
> + "from $package ($filename) line $line. You are likely using ".
> + "a plugin that is not compatible with SpamAssasin 4.0.0.";
> + }
>
> return if $self->{main}->{resolver}->{no_resolver};
> $self->_start_lookup(@_);
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm Fri May 13 06:06:33 2022
> @@ -110,6 +110,8 @@ sub do_rbl_lookup {
> dbg("dns: launching rule %s, set %s, type %s, %s", $rule, $set, $type,
> defined $subtest ? "subtest $subtest" : 'no subtest');
>
> + $self->rule_pending($rule); # mark async
> +
> my $ent = {
> rulename => $rule,
> type => "DNSBL",
> @@ -126,6 +128,8 @@ sub do_rbl_lookup {
> sub do_dns_lookup {
> my ($self, $rule, $type, $host) = @_;
>
> + $self->rule_pending($rule); # mark async
> +
> my $ent = {
> rulename => $rule,
> type => "DNSBL",
> @@ -222,6 +226,20 @@ sub process_dnsbl_result {
> my $question = ($pkt->question)[0];
> return if !$question;
>
> + my $rulename = $ent->{rulename};
> +
> + # Mark rule ready for meta rules, but only if this was the last lookup
> + # pending, rules can have many lookups launched for different IPs
> + if (!$self->get_async_pending_rules($rulename)) {
> + $self->rule_ready($rulename);
> + # Mark depending check_rbl_sub rules too
> + if (exists $self->{rbl_subs}{$ent->{set}}) {
> + foreach (@{$self->{rbl_subs}{$ent->{set}}}) {
> + $self->rule_ready($_->[1]);
> + }
> + }
> + }
> +
> # DNSBL tests are here
> foreach my $answer ($pkt->answer) {
> next if !$answer;
> @@ -256,17 +274,18 @@ sub process_dnsbl_result {
> # check_rbl tests
> if (defined $ent->{subtest}) {
> if ($self->check_subtest($rdatastr, $ent->{subtest})) {
> - $self->dnsbl_hit($ent->{rulename}, $question, $answer);
> + $self->dnsbl_hit($rulename, $question, $answer);
> }
> } else {
> - $self->dnsbl_hit($ent->{rulename}, $question, $answer);
> + $self->dnsbl_hit($rulename, $question, $answer);
> }
>
> # check_rbl_sub tests
> - if (defined $self->{rbl_subs}{$ent->{set}}) {
> + if (exists $self->{rbl_subs}{$ent->{set}}) {
> $self->process_dnsbl_set($ent->{set}, $question, $answer, $rdatastr);
> }
> }
> +
> return 1;
> }
>
> @@ -685,9 +704,9 @@ sub register_async_rule_finish {}
> sub mark_all_async_rules_complete {}
> sub is_rule_complete {}
>
> -# Return number of pending lookups for a rule,
> +# Return number of pending DNS lookups for a rule,
> # or list all of rules still pending
> -sub get_pending_lookups {
> +sub get_async_pending_rules {
> my ($self, $rule) = @_;
> if (defined $rule) {
> return 0 if !exists $self->{async}->{pending_rules}{$rule};
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Fri May 13 06:06:33 2022
> @@ -3104,6 +3104,9 @@ sub got_hit {
> $score = $params{defscore} if !defined $score;
> }
>
> + # Make sure rule is marked ready in tests_pending also
> + $self->{tests_pending}->{$rule} = 0;
> +
> # adding a hit does nothing if we don't have a score -- we probably
> # shouldn't have run it in the first place
> if (!$score) {
> @@ -3170,20 +3173,25 @@ sub got_hit {
>
> =item $status->rule_pending ($rulename)
>
> -Register a pending rule. Must be called from rules eval-function, if the
> -result can arrive later than when exiting the function (async lookups).
> -This is not required if eval uses $status->{async}->bgsend_and_start_lookup(),
> -as it automatically registers a pending rule and also marks it ready when the
> -result callback is run.
> -
> -$status->rule_ready($rulename) or $status->got_hit(...) must be called when
> -the result has arrived. If these are not used, it can break dynamic meta rule
> -evaluation. This does not apply when bgsend_and_start_lookup() was used, as
> -mentioned above.
> +Register a pending rule. Must be called if the rules result is evaluated
> +later than when exiting it's eval-function (e.g. async lookups). This is
> +required for meta rules that depend on this rule to evaluate correctly.
> +
> +It is not strictly required when $status->{async}->bgsend_and_start_lookup()
> +is used, as that automatically tracks pending DNS rules and also marks them
> +ready when the result callback is run. But when a plugin has dummy stub
> +eval-function (examples: check_uridnsbl, check_rbl_sub), then using
> +rule_pending() and rule_ready() is necessary. Otherwise in some cases the
> +eval-function could be run before any lookups are launched, and the rule
> +would be marked ready too early.
> +
> +$status->rule_ready($rulename) or $status->got_hit($rulename, ...) must be
> +called when the result has arrived. If these are not used, then any meta
> +rules depending on the rule might not evaluate at all.
>
> =cut
>
> -# === Bug 7735 notes ===
> +# === Bug 7735 technical notes ===
> #
> # Rule readiness status for dynamic meta rules evaluation (Plugins / Check /
> # do_meta_tests) is always tracked by $pms->{tests_already_hit}->{$rule}.
> @@ -3191,9 +3199,12 @@ mentioned above.
> #
> # The public rule_pending() / rule_ready() function pair is meant to be used
> # for any async rules, and they additionally track rule status in
> -# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. So any
> -# rule_pending() call must always be accompanied later by rule_ready() or
> -# got_hit(). These are not required when bgsend_and_start_lookup() is used.
> +# $pms->{tests_pending}->{$rule}, which do_meta_tests also checks. Any
> +# rule_pending() call _must_ be accompanied later by rule_ready() or
> +# got_hit(). But as bgsend_and_start_lookup() automatically tracks rules
> +# too, most of legacy plugins not using these should work correctly - only
> +# in some corner cases where a dummy stub eval-function is called early by
> +# accident (maybe due to adjusted rule priority) things might break.
> #
> # Any rule regardless of what it is must always be marked ready via
> # $pms->{tests_already_hit} for meta evaluation to work, even if no
> @@ -3201,38 +3212,43 @@ mentioned above.
> # use $pms->{tests_already_hit} directly for marking, as is done for example
> # by Check.pm for all basic body/header/etc rules (look for
> # $hitsptr->{q{'.$rulename.'}} ||= 0). As such it's also always safe to
> -# call rule_ready() without rule_pending().
> +# call rule_ready() without rule_pending(), it achieves the same.
>
> sub rule_pending {
> my ($self, $rule) = @_;
>
> + # Ignore if called after rule_pending(), rule_ready() or got_hit(), they
> + # are the only ones that make $self->{tests_pending}->{$rule} exist
> + return if exists $self->{tests_pending}->{$rule};
> +
> + # It's possible that tests_already_hit == 0, if some stub eval call was
> + # run early. It's ok to clear this and pretend it was never run. We now
> + # rely exclusively on tests_pending and wait for rule_ready() or got_hit()
> + delete $self->{tests_already_hit}->{$rule} if !$self->{tests_already_hit}->{$rule};
> $self->{tests_pending}->{$rule} = 1;
> -
> - if (exists $self->{tests_already_hit}->{$rule}) {
> - # Only clear result if not hit
> - if ($self->{tests_already_hit}->{$rule} == 0) {
> - delete $self->{tests_already_hit}->{$rule};
> - }
> - }
> }
>
> =item $status->rule_ready ($rulename)
>
> -Mark a previously marked $status->rule_pending() rule ready. Alternatively
> -$status->got_hit() will also mark rule ready. If these are not used, it can
> -break dynamic meta rule evaluation.
> +Mark a rule ready, so it can be considered for meta rule evaluation. Any
> +rule regardless of type must always be marked ready when it's finished,
> +otherwise any meta rules that depend on it might not evaluate. If
> +$status->rule_pending() was called, then a $stats->rule_ready() call must be
> +done, alternatively $status->got_hit() will also mark rule ready.
>
> =cut
>
> sub rule_ready {
> my ($self, $rule) = @_;
>
> - if ($self->get_pending_lookups($rule)) {
> - # Can't be ready if there are pending lookups, ignore for now.
> + if ($self->get_async_pending_rules($rule)) {
> + # Can't be ready if there are pending DNS lookups, ignore for now.
> + # Make sure tests_pending exist, should not accept rule_pending() anymore
> + $self->{tests_pending}->{$rule} ||= 0;
> return;
> }
>
> - delete $self->{tests_pending}->{$rule};
> + $self->{tests_pending}->{$rule} = 0;
> $self->{tests_already_hit}->{$rule} ||= 0;
> }
>
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/AskDNS.pm Fri May 13 06:06:33 2022
> @@ -433,13 +433,14 @@ sub launch_queries {
> next;
> }
> dbg("askdns: launching query ($rulename): $query");
> - $pms->{async}->bgsend_and_start_lookup(
> + my $ret = $pms->{async}->bgsend_and_start_lookup(
> $query, $arule->{q_type}, undef,
> { rulename => $rulename, type => 'AskDNS' },
> sub { my ($ent,$pkt) = @_;
> $self->process_response_packet($pms, $ent, $pkt, $rulename) },
> master_deadline => $pms->{master_deadline}
> );
> + $pms->rule_ready($rulename) if !$ret; # mark ready if nothing launched
> }
> }
>
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Check.pm Fri May 13 06:06:33 2022
> @@ -54,8 +54,8 @@ sub check_main {
> my $pms = $args->{permsgstatus};
> $would_log_rules_all = would_log('dbg', 'rules-all') == 2;
>
> - # Make AsyncLoop wait permission for launching queries
> - $pms->{async}->{wait_launch} = 1;
> + # Make AsyncLoop wait launch_queue() for launching queries
> + $pms->{async}->start_queue();
>
> my $suppl_attrib = $pms->{msg}->{suppl_attrib};
> if (ref $suppl_attrib && ref $suppl_attrib->{rule_hits}) {
> @@ -129,7 +129,6 @@ sub check_main {
> # unneeded DNS queries.
> if ($do_dns && !$rbls_running && $priority >= -100) {
> $rbls_running = 1;
> - $pms->{async}->{wait_launch} = 0; # permission granted
> $pms->{async}->launch_queue(); # check if something was queued
> $self->run_rbl_eval_tests($pms);
> $self->{main}->call_plugins ("check_dnsbl", { permsgstatus => $pms });
> @@ -289,8 +288,8 @@ sub do_meta_tests {
> my $h = $pms->{tests_already_hit};
> my $retry;
>
> - # Get pending async rule list
> - my %pl = map { $_ => 1 } $pms->get_pending_lookups();
> + # Get pending DNS async rule list
> + my %pl = map { $_ => 1 } $pms->get_async_pending_rules();
>
> RULE:
> foreach my $rulename (keys %$mp) {
> @@ -323,6 +322,7 @@ sub finish_meta_tests {
> return if $self->{am_compiling}; # nothing to compile here
>
> my $mp = $pms->{meta_pending};
> + my $tp = $pms->{tests_pending};
> my $md = $pms->{conf}->{meta_dependencies};
> my $mt = $pms->{conf}->{meta_tests};
> my $h = $pms->{tests_already_hit};
> @@ -334,7 +334,7 @@ RULE:
> my %unrun;
> # Meta is not ready if some dependency has not run yet
> foreach my $deprule (@{$md->{$rulename}||[]}) {
> - if (!exists $h->{$deprule}) {
> + if (!exists $h->{$deprule} || $tp->{$deprule}) {
> # Record all unrun deps for second meta evaluation
> $unrun{$deprule} = 1;
> }
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/DNSEval.pm Fri May 13 06:06:33 2022
> @@ -150,13 +150,14 @@ sub set_config {
>
> sub finish_parsing_start {
> my ($self, $opts) = @_;
> + my $conf = $opts->{conf};
>
> # Adjust priority -100 to launch early
> # Find rulenames from eval_to_rule mappings
> foreach my $evalfunc (@{$self->{'evalrules'}}) {
> - foreach (@{$opts->{conf}->{eval_to_rule}->{$evalfunc}||[]}) {
> + foreach (@{$conf->{eval_to_rule}->{$evalfunc}||[]}) {
> dbg("dnseval: adjusting rule $_ priority to -100");
> - $opts->{conf}->{priority}->{$_} = -100;
> + $conf->{priority}->{$_} = -100;
> }
> }
> }
> @@ -165,34 +166,33 @@ sub finish_parsing_start {
> # directly as part of PMS
> sub check_start {
> my ($self, $opts) = @_;
> + my $pms = $opts->{permsgstatus};
>
> foreach(@{$self->{'evalrules'}}) {
> - $opts->{'permsgstatus'}->register_plugin_eval_glue($_);
> + $pms->register_plugin_eval_glue($_);
> }
>
> # Initialize check_rbl_sub tests
> - $self->init_rbl_subs($opts->{'permsgstatus'});
> + $self->_init_rbl_subs($pms);
> }
>
> -sub init_rbl_subs {
> +sub _init_rbl_subs {
> my ($self, $pms) = @_;
> -
> - return if $pms->{rbl_subs};
> + my $conf = $pms->{conf};
>
> # Very hacky stuff and direct rbl_evals usage for now, TODO rewrite everything
> - foreach my $rule (@{$pms->{conf}->{eval_to_rule}->{check_rbl_sub}}) {
> - next if !exists $pms->{conf}->{rbl_evals}->{$rule};
> - next if !$pms->{conf}->{scores}->{$rule};
> + foreach my $rule (@{$conf->{eval_to_rule}->{check_rbl_sub}||[]}) {
> + next if !exists $conf->{rbl_evals}->{$rule};
> + next if !$conf->{scores}->{$rule};
> # rbl_evals is [$function,[@args]]
> - my $args = $pms->{conf}->{rbl_evals}->{$rule}->[1];
> - my $set = $args->[0];
> - my $subtest = $args->[1];
> + my $args = $conf->{rbl_evals}->{$rule}->[1];
> + my ($set, $subtest) = @$args;
> if (!defined $subtest) {
> warn("dnseval: missing subtest for rule $rule\n");
> next;
> }
> if ($subtest =~ /^sb:/) {
> - info("dnseval: ignored $rule, SenderBase rules are deprecated");
> + warn("dnseval: ignored $rule, SenderBase rules are deprecated\n");
> next;
> }
> # Compile as regex if not pure ip/bitmask (same check in process_dnsbl_result)
> @@ -206,6 +206,7 @@ sub init_rbl_subs {
> }
> dbg("dnseval: initialize check_rbl_sub for rule $rule, set $set, subtest $subtest");
> push @{$pms->{rbl_subs}{$set}}, [$subtest, $rule];
> + $pms->rule_pending($rule); # mark async, rule_ready() done in Dns/process_dnsbl_result
> }
> }
>
> @@ -283,7 +284,7 @@ sub check_rbl_accreditor {
> $self->message_accreditor_tag($pms);
> }
> if ($pms->{accreditor_tag}->{$accreditor}) {
> - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
> + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
> }
> return 0;
> }
> @@ -324,7 +325,7 @@ sub message_accreditor_tag {
> $pms->{accreditor_tag} = \%acctags;
> }
>
> -sub check_rbl_backend {
> +sub _check_rbl_backend {
> my ($self, $pms, $rule, $set, $rbl_server, $type, $subtest) = @_;
>
> return if !exists $pms->{dnseval_ips}; # no untrusted ips
> @@ -406,14 +407,12 @@ sub check_rbl_backend {
> return 0;
> }
>
> - $pms->rule_pending($rule); # mark async
> -
> dbg("dnseval: only inspecting the following IPs: ".join(", ", @ips));
>
> foreach my $ip (@ips) {
> - my $revip = reverse_ip_address($ip);
> - $pms->do_rbl_lookup($rule, $set, $type,
> - $revip.'.'.$rbl_server, $subtest) if defined $revip;
> + if (defined(my $revip = reverse_ip_address($ip))) {
> + $pms->do_rbl_lookup($rule, $set, $type, $revip.'.'.$rbl_server, $subtest)
> + }
> }
>
> # note that results are not handled here, hits are handled directly
> @@ -427,7 +426,7 @@ sub check_rbl {
> return 0 if $self->{main}->{conf}->{skip_rbl_checks};
> return 0 if !$pms->is_dns_available();
>
> - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
> + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'A', $subtest);
> }
>
> sub check_rbl_txt {
> @@ -436,12 +435,12 @@ sub check_rbl_txt {
> return 0 if $self->{main}->{conf}->{skip_rbl_checks};
> return 0 if !$pms->is_dns_available();
>
> - $self->check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest);
> + $self->_check_rbl_backend($pms, $rule, $set, $rbl_server, 'TXT', $subtest);
> }
>
> sub check_rbl_sub {
> my ($self, $pms, $rule, $set, $subtest) = @_;
> - # just a dummy, check_start / init_rbl_subs handles the subs
> + # just a dummy, _init_rbl_subs/do_rbl_lookup handles the subs
> $pms->rule_pending($rule); # mark async
> return 0;
> }
> @@ -454,8 +453,6 @@ sub check_rbl_from_host {
> return 0 if $self->{main}->{conf}->{skip_rbl_checks};
> return 0 if !$pms->is_dns_available();
>
> - $pms->rule_pending($rule); # mark async
> -
> $self->_check_rbl_addresses($pms, $rule, $set, $rbl_server,
> $subtest, $pms->all_from_addrs());
> }
> @@ -551,10 +548,11 @@ sub check_rbl_ns_from {
>
> dbg("dnseval: checking NS for host $domain");
>
> - my $key = "NS:" . $domain;
> + $pms->rule_pending($rule); # mark async
> +
> my $obj = { dom => $domain, rule => $rule, set => $set, rbl_server => $rbl_server, subtest => $subtest };
> my $ent = {
> - rulename => $rule, key => $key, zone => $domain, obj => $obj, type => "URI-NS",
> + rulename => $rule, zone => $domain, obj => $obj, type => "URI-NS",
> };
> # dig $dom ns
> $ent = $pms->{async}->bgsend_and_start_lookup(
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/OneLineBodyRuleType.pm Fri May 13 06:06:33 2022
> @@ -66,6 +66,21 @@ sub check_start {
> }
> }
>
> +sub check_cleanup {
> + my ($self, $params) = @_;
> + my $pms = $params->{permsgstatus};
> + my $hitsptr = $pms->{tests_already_hit};
> + my $scoresptr = $pms->{conf}->{scores};
> +
> + # Force all body rules ready for meta rules. Need to do it here in
> + # cleanup, because the body is scanned per line instead of per rule
> + if ($pms->{conf}->{skip_body_rules}) {
> + foreach (keys %{$pms->{conf}->{skip_body_rules}}) {
> + $hitsptr->{$_} ||= 0 if $scoresptr->{$_};
> + }
> + }
> +}
> +
> ###########################################################################
>
> 1;
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/P595Body.pm Fri May 13 06:06:33 2022
> @@ -117,6 +117,11 @@ sub check_rules_at_priority {
> $self->{one_line_body}->check_rules_at_priority($params);
> }
>
> +sub check_cleanup {
> + my ($self, $params) = @_;
> + $self->{one_line_body}->check_cleanup($params);
> +}
> +
> ###########################################################################
>
> sub run_body_fast_scan {
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Rule2XSBody.pm Fri May 13 06:06:33 2022
> @@ -197,6 +197,11 @@ sub check_rules_at_priority {
> $self->{one_line_body}->check_rules_at_priority($params);
> }
>
> +sub check_cleanup {
> + my ($self, $params) = @_;
> + $self->{one_line_body}->check_cleanup($params);
> +}
> +
> ###########################################################################
>
> sub run_body_fast_scan {
>
> Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm (original)
> +++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Fri May 13 06:06:33 2022
> @@ -378,6 +378,8 @@ sub check_dnsbl {
> foreach my $rulename (keys %{$conf->{uridnsbls}}) {
> next if !$conf->{scores}->{$rulename};
>
> + $pms->rule_pending($rulename); # mark async
> +
> my $rulecf = $conf->{uridnsbls}->{$rulename};
> my %tfl = map { ($_,1) } split(/\s+/, $conf->{tflags}->{$rulename}||'');
>
>
> Modified: spamassassin/trunk/t/basic_meta2.t
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta2.t?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/t/basic_meta2.t (original)
> +++ spamassassin/trunk/t/basic_meta2.t Fri May 13 06:06:33 2022
> @@ -5,7 +5,7 @@ use lib 't';
> use SATest; sa_t_init("basic_meta2");
>
> use Test::More;
> -plan tests => 15;
> +plan tests => 20;
>
> # ---------------------------------------------------------------------------
>
> @@ -19,6 +19,11 @@ plan tests => 15;
> q{ TEST_META_7 } => '',
> q{ TEST_META_A } => '',
> q{ TEST_META_B } => '',
> + q{ TEST_META_C } => '',
> + q{ TEST_META_D } => '',
> + q{ TEST_META_E } => '',
> + q{ TEST_META_F } => '',
> + q{ TEST_META_G } => '',
> );
>
> %anti_patterns = (
> @@ -80,6 +85,21 @@ tstlocalrules (qq{
> # local_tests_only
> meta TEST_META_B NONEXISTINGRULE || local_tests_only
>
> + # complex metas with different priorities
> + body __BAR_5 /a/
> + priority __BAR_5 -1000
> + body __BAR_6 /b/
> + priority __BAR_6 0
> + body __BAR_7 /c/
> + priority __BAR_7 1000
> + meta TEST_META_C __BAR_5 && __BAR_6 && __BAR_7
> + meta TEST_META_D __BAR_5 && __BAR_6 && TEST_META_C
> + priority TEST_META_D -2000
> + meta TEST_META_E __BAR_6 && __BAR_7 && TEST_META_D
> + meta TEST_META_F __BAR_5 && __BAR_7 && TEST_META_E
> + priority TEST_META_F 2000
> + meta TEST_META_G TEST_META_C && TEST_META_D && TEST_META_E && TEST_META_F
> +
> });
>
> sarun ("-L -t < data/nice/001 2>&1", \&patterns_run_cb);
>
> Modified: spamassassin/trunk/t/basic_meta_net.t
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/basic_meta_net.t?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/t/basic_meta_net.t (original)
> +++ spamassassin/trunk/t/basic_meta_net.t Fri May 13 06:06:33 2022
> @@ -8,79 +8,87 @@ use Test::More;
> plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
> plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
>
> -plan tests => 20;
> +plan tests => 32;
>
> # ---------------------------------------------------------------------------
>
>
> %patterns = (
> - q{ X_META_POS4 } => '',
> + q{ 1.0 X_LOCAL_TESTS } => '',
> );
> %anti_patterns = (
> - q{ X_URIBL_A } => '',
> - q{ X_ASKDNS } => '',
> - q{ X_META_POS1 } => '',
> - q{ X_META_POS2 } => '',
> - q{ X_META_POS3 } => '',
> - q{ X_META_NEG1 } => '',
> - q{ X_META_NEG2 } => '',
> - q{ X_META_NEG3 } => '',
> - q{ X_META_NEG4 } => '',
> + q{ 1.0 X_URIBL_A } => '',
> + q{ 1.0 X_ASKDNS } => '',
> + q{ 1.0 X_DNSBL_TEST } => '',
> + q{ 1.0 X_DNSBL_SUB } => '',
> + q{ 1.0 X_META_POS1 } => '',
> + q{ 1.0 X_META_POS2 } => '',
> + q{ 1.0 X_META_POS3 } => '',
> + q{ 1.0 X_META_POS4 } => '',
> + q{ 1.0 X_META_POS5 } => '',
> + q{ 1.0 X_META_NEG1 } => '',
> + q{ 1.0 X_META_NEG2 } => '',
> + q{ 1.0 X_META_NEG3 } => '',
> + q{ 1.0 X_META_NEG4 } => '',
> + q{ 1.0 X_META_NEG5 } => '',
> + q{ 1.0 X_LOCAL_NEG } => '',
> );
>
> -#
> -# Nothing should hit with a failed lookup
> -#
> -
> -tstlocalrules (qq{
> - # Force DNS queries to fail/timeout
> - rbl_timeout 2 1
> - dns_server 240.0.0.240
> -
> +my $common_rules = q{
> urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2
> body X_URIBL_A eval:check_uridnsbl('X_URIBL_A')
> tflags X_URIBL_A net
>
> askdns X_ASKDNS spamassassin.org TXT /./
>
> + header X_DNSBL_TEST eval:check_rbl('test', 'dnsbltest.spamassassin.org.')
> + tflags X_DNSBL_TEST net
> +
> + header X_DNSBL_SUB eval:check_rbl_sub('test', '2')
> + tflags X_DNSBL_SUB net
> +
> meta X_META_POS1 X_URIBL_A
> meta X_META_POS2 X_ASKDNS
> - meta X_META_POS3 X_URIBL_A || X_ASKDNS
> + meta X_META_POS3 X_DNSBL_TEST
> + meta X_META_POS4 X_DNSBL_SUB
> + meta X_META_POS5 X_URIBL_A || X_ASKDNS || X_DNSBL_TEST || X_DNSBL_SUB
>
> meta X_META_NEG1 !X_URIBL_A
> meta X_META_NEG2 !X_ASKDNS
> - meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS
> + meta X_META_NEG3 !X_DNSBL_TEST
> + meta X_META_NEG4 !X_DNSBL_SUB
> + meta X_META_NEG5 !X_URIBL_A || !X_ASKDNS || !X_DNSBL_TEST || !X_DNSBL_SUB
> +};
> +
> +#
> +# Nothing should hit with a timed out lookup
> +#
> +
> +tstlocalrules (qq{
> + # Force DNS queries to fail/timeout
> + rbl_timeout 2 1
> + dns_server 240.0.0.240
> +
> + $common_rules
>
> # local_tests_only
> - meta X_META_NEG4 local_tests_only
> - meta X_META_POS4 !local_tests_only
> + meta X_LOCAL_TESTS !local_tests_only
> + meta X_LOCAL_NEG local_tests_only
> });
>
> -sarun ("-t < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb);
> +sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
> ok_all_patterns();
>
> #
> -# Local only, nothing should hit as nothing is queried
> +# Local tests only, nothing should hit as nothing is queried
> #
>
> tstlocalrules (qq{
> - urirhssub X_URIBL_A dnsbltest.spamassassin.org. A 2
> - body X_URIBL_A eval:check_uridnsbl('X_URIBL_A')
> - tflags X_URIBL_A net
> -
> - askdns X_ASKDNS spamassassin.org TXT /./
> -
> - meta X_META_POS1 X_URIBL_A
> - meta X_META_POS2 X_ASKDNS
> - meta X_META_POS3 X_URIBL_A || X_ASKDNS
> -
> - meta X_META_NEG1 !X_URIBL_A
> - meta X_META_NEG2 !X_ASKDNS
> - meta X_META_NEG3 !X_URIBL_A || !X_ASKDNS
> + $common_rules
>
> # local_tests_only
> - meta X_META_POS4 local_tests_only
> - meta X_META_NEG4 !local_tests_only
> + meta X_LOCAL_TESTS local_tests_only
> + meta X_LOCAL_NEG !local_tests_only
> });
>
> sarun ("-t -L < data/spam/dnsbl.eml", \&patterns_run_cb);
>
> Modified: spamassassin/trunk/t/dnsbl.t
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/dnsbl.t?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/t/dnsbl.t (original)
> +++ spamassassin/trunk/t/dnsbl.t Fri May 13 06:06:33 2022
> @@ -6,7 +6,10 @@ use SATest; sa_t_init("dnsbl");
> use Test::More;
> plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
> plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
> -plan tests => 21;
> +
> +# run many times to catch some random natured failures
> +my $iterations = 5;
> +plan tests => 22 * $iterations;
>
> # ---------------------------------------------------------------------------
> # bind configuration currently used to support this test
> @@ -71,6 +74,7 @@ EOF
> q{'1.0 DNSBL_TXT_MISS'} => '',
> q{'1.0 DNSBL_TEST_WHITELIST_MISS'} => '',
> q{'14.35.17.212.untrusted.dnsbltest.spamassassin.org'} => '',
> + q{/rules-all: unrun dependencies [^\n]+ (?:__|META_)?DNSBL_/} => '',
> );
>
> tstprefs("
> @@ -148,6 +152,9 @@ priority DNSBL_TEST_RELAY 2000
>
> ");
>
> -sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
> -ok_all_patterns();
> +for (1 .. $iterations) {
> + # rules-all debug needed for unrun check
> + sarun ("-t -D rules-all < data/spam/dnsbl.eml 2>&1", \&patterns_run_cb);
> + ok_all_patterns();
> +}
>
>
> Modified: spamassassin/trunk/t/sa_compile.t
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/sa_compile.t?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/t/sa_compile.t (original)
> +++ spamassassin/trunk/t/sa_compile.t Fri May 13 06:06:33 2022
> @@ -54,35 +54,36 @@ body FOO5 /éèæ©/
> body FOO6 /éè(?:xyz|æ©)/
> body FOO7 /\xe9\x87\x91\xe8\x9e\x8d\xe6\xa9\x9f/
> body FOO8 /.\x87(?:\x91|\x00)[\xe8\x00]\x9e\x8d\xe6\xa9\x9f/
> +# Test that meta rules work for sa-compiled body rules
> +# (loosely related to Bug 7987)
> +meta META1 FOO1 && FOO2 && FOO3 && FOO4
> +meta META2 FOO5 && FOO6 && FOO7 && FOO8
> ');
>
> # ensure we don't use compiled rules
> untaint_system("rm -rf $instdir/var/spamassassin/compiled");
>
> %patterns = (
> - '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO',
> + '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '',
> );
> %anti_patterns = (
> '/ zoom: able to use /', '',
> );
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> +
> %patterns = (
> - '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO',
> + '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '',
> );
> %anti_patterns = (
> '/ zoom: able to use /', '',
> );
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
>
> # -------------------------------------------------------------------
>
> @@ -92,27 +93,24 @@ $scr = "$instdir/$temp_binpath/spamassas
> $scr_localrules_args = $scr_cf_args = ""; # use the default rules dir, from our "install"
>
> %patterns = (
> - q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use',
> - '/ check: tests=FOO1,FOO2,FOO3,FOO4\n/', 'FOO',
> + q{ zoom: able to use 5/5 'body_0' compiled rules }, '',
> + '/ check: tests=FOO1,FOO2,FOO3,FOO4,META1\n/', '',
> );
> %anti_patterns = ();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/001 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> +
> %patterns = (
> - q{ zoom: able to use 5/5 'body_0' compiled rules }, 'able-to-use',
> - '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8\n/', 'FOO',
> + q{ zoom: able to use 5/5 'body_0' compiled rules }, '',
> + '/ check: tests=FOO4,FOO5,FOO6,FOO7,FOO8,META2\n/', '',
> );
> %anti_patterns = ();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 1' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
> -ok sarun ("-D all,rules-all -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> +ok sarun ("-D check,zoom -L -t --cf 'normalize_charset 0' < $cwd/data/spam/unicode1 2>&1", \&patterns_run_cb);
> ok_all_patterns();
> -clear_pattern_counters();
>
> # -------------------------------------------------------------------
>
>
> Modified: spamassassin/trunk/t/uribl.t
> URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/uribl.t?rev=1900849&r1=1900848&r2=1900849&view=diff
> ==============================================================================
> --- spamassassin/trunk/t/uribl.t (original)
> +++ spamassassin/trunk/t/uribl.t Fri May 13 06:06:33 2022
> @@ -6,7 +6,10 @@ use SATest; sa_t_init("uribl");
> use Test::More;
> plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
> plan skip_all => "Can't use Net::DNS Safely" unless can_use_net_dns_safely();
> -plan tests => 10;
> +
> +# run many times to catch some random natured failures
> +my $iterations = 5;
> +plan tests => 10 * $iterations;
>
> # ---------------------------------------------------------------------------
>
> @@ -68,6 +71,8 @@ tstlocalrules(q{
>
> });
>
> -ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
> -ok_all_patterns();
> +for (1 .. $iterations) {
> + ok sarun ("-t < data/spam/dnsbl.eml", \&patterns_run_cb);
> + ok_all_patterns();
> +}
>
>