You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by mm...@apache.org on 2007/10/17 02:04:54 UTC
svn commit: r585292 - in /spamassassin/trunk/lib/Mail/SpamAssassin:
AsyncLoop.pm Conf.pm Dns.pm Plugin/ASN.pm Plugin/URIDNSBL.pm
Author: mmartinec
Date: Tue Oct 16 17:04:48 2007
New Revision: 585292
URL: http://svn.apache.org/viewvc?rev=585292&view=rev
Log:
AsyncLoop: replaces sharp clipping on a lower timeout value with a
smooth quadratic transition curve from an initial to a minimal timeout
value; updated documentation accordingly. Adjust actual timeout value
to timer resolution (ceiling) if Time::HiRes module is not available.
Conf+AsyncLoop: to a configuration variable rbl_timeout add two optional
parameters: a minimal timeout value, and a zone name if a timeout
specification should only apply to the specified zone or its subdomains;
update documentation accordingly.
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm?rev=585292&r1=585291&r2=585292&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/AsyncLoop.pm Tue Oct 16 17:04:48 2007
@@ -45,8 +45,15 @@
# Load Time::HiRes if it's available
BEGIN {
- eval { require Time::HiRes };
- Time::HiRes->import( qw(time) ) unless $@;
+ use vars qw($timer_resolution);
+ eval {
+ require Time::HiRes or die "Error loading Time::HiRes: $@, $!";
+ Time::HiRes->import( qw(time CLOCK_REALTIME) );
+ $timer_resolution = Time::HiRes::clock_getres(CLOCK_REALTIME());
+ 1;
+ } or do {
+ $timer_resolution = 1; # Perl's builtin timer ticks at one second
+ };
}
#############################################################################
@@ -123,27 +130,33 @@
The code reference will be called with one argument, the C<$ent> object.
+=item zone (optional)
+
+A zone specification (typically a DNS zone name - e.g. host, domain, or RBL)
+which is used as a key to look up per-zone settings (currently both timeouts).
+
=item timeout_initial (optional)
An initial value of elapsed time for which we are willing to wait for a
response (time in seconds, floating point value is allowed). When elapsed
time since a query started exceeds the timeout value and there are no other
-queries to wait for, the query is aborted. The actual timeout value may
-shrink dynamically by some factor from a timeout_initial value, the factor
-is derived from a number of queries that have already completed, but the
-value can not fall below timeout_min (see next parameter).
+queries to wait for, the query is aborted. The actual timeout value ranges
+from timeout_initial and gradually approaches timeout_min (see next parameter)
+as the number of already completed queries approaches the number of all
+queries started.
If a caller does not explicitly provide this parameter or its value is
undefined, a default initial timeout value is settable by a configuration
variable rbl_timeout.
-If value of the timeout_initial parameter is already below timeout_min,
-it is pushed up to timeout_min.
+If a value of the timeout_initial parameter is below timeout_min, the initial
+timeout is set to timeout_min.
=item timeout_min (optional)
-A lower bound (in seconds) below which the timeout value can not be
-reduced by a dynamic timeout shrinkage formula. Defaults to 1 second.
+A lower bound (in seconds) to which the actual timeout approaches as the
+number of queries completed approaches the number of all queries started.
+Defaults to 0.2 * timeout_initial.
=back
@@ -164,11 +177,40 @@
$ent->{status} = 'STARTED';
$ent->{start_time} = $now if !defined $ent->{start_time};
- $ent->{timeout_min} = 1 if !defined $ent->{timeout_min}; # limits shrinkage
- my $timeout = $ent->{timeout_initial};
- $timeout = $self->{main}->{conf}->{rbl_timeout} if !defined $timeout;
- $timeout = $ent->{timeout_min} if $timeout < $ent->{timeout_min};
- $ent->{timeout_initial} = $timeout;
+ # are there any applicable per-zone settings?
+ my $zone = $ent->{zone};
+ my $settings; # a ref to a by-zone or to global settings
+ my $conf_by_zone = $self->{main}->{conf}->{by_zone};
+ if (defined $zone && $conf_by_zone) {
+ # dbg("async: searching for by_zone settings for $zone");
+ $zone =~ s/^\.//; $zone =~ s/\.\z//; # strip leading and trailing dot
+ for (;;) { # 2.10.example.com, 10.example.com, example.com, com, ''
+ if (exists $conf_by_zone->{$zone}) {
+ $settings = $conf_by_zone->{$zone};
+ dbg("async: applying by_zone settings for $zone");
+ last;
+ } elsif ($zone eq '') {
+ last;
+ } else { # strip one level, careful with address literals
+ $zone = ($zone =~ /^( (?: [^.] | \[ (?: \\. | [^\]\\] )* \] )* )
+ \. (.*) \z/xs) ? $2 : '';
+ }
+ }
+ }
+
+ my $t_init = $ent->{timeout_initial}; # application-specified has precedence
+ $t_init = $settings->{rbl_timeout} if $settings && !defined $t_init;
+ $t_init = $self->{main}->{conf}->{rbl_timeout} if !defined $t_init;
+ $t_init = 0 if !defined $t_init; # last-resort default, just in case
+
+ my $t_end = $ent->{timeout_min}; # application-specified has precedence
+ $t_end = $settings->{rbl_timeout_min} if $settings && !defined $t_end;
+ $t_end = 0.2 * $t_init if !defined $t_end;
+ $t_end = 0 if $t_end < 0; # just in case
+
+ $t_init = $t_end if $t_init < $t_end;
+ $ent->{timeout_initial} = $t_init;
+ $ent->{timeout_min} = $t_end;
$ent->{display_id} = # identifies entry in debug logging and similar
join(", ", grep { defined }
@@ -265,28 +307,29 @@
if (defined $timeout && $timeout > 0 &&
%$pending && $self->{total_queries_started} > 0)
{
- # shrink 'select' timeout if a caller specified unnecessarily long
- # value, beyond the latest deadline of any outstanding request;
+ # shrink a 'select' timeout if a caller specified unnecessarily long
+ # value beyond the latest deadline of any outstanding request;
# can save needless wait time (up to 1 second in harvest_dnsbl_queries)
- my $timeout_shrink_factor =
- 1 - 0.7 * ( ($self->{total_queries_completed} /
- $self->{total_queries_started}) ** 2 );
+ my $r = $self->{total_queries_completed} / $self->{total_queries_started};
+ my $r2 = $r * $r; # 0..1
my $max_deadline;
while (my($key,$ent) = each %$pending) {
- my $dt = $ent->{timeout_initial} * $timeout_shrink_factor;
- # don't shrink below a timeout's lower bound
- $dt = $ent->{timeout_min} if $dt < $ent->{timeout_min};
+ my $t_init = $ent->{timeout_initial};
+ my $dt = $t_init - ($t_init - $ent->{timeout_min}) * $r2;
my $deadline = $ent->{start_time} + $dt;
- if (!defined $max_deadline || $deadline > $max_deadline) {
- $max_deadline = $deadline;
- }
+ $max_deadline = $deadline if $deadline > $max_deadline;
}
- my $sufficient_timeout = $max_deadline - $now;
- $sufficient_timeout = 0 if $sufficient_timeout < 0;
- if (defined $max_deadline && $timeout > $sufficient_timeout) {
- dbg("async: reducing select timeout from %.1f to %.1f s",
- $timeout, $sufficient_timeout);
- $timeout = $sufficient_timeout;
+ if (defined $max_deadline) {
+ # adjust to timer resolution, only deals with 1s and with fine resolution
+ $max_deadline = 1 + int $max_deadline
+ if $timer_resolution == 1 && $max_deadline > int $max_deadline;
+ my $sufficient_timeout = $max_deadline - $now;
+ $sufficient_timeout = 0 if $sufficient_timeout < 0;
+ if ($timeout > $sufficient_timeout) {
+ dbg("async: reducing select timeout from %.1f to %.1f s",
+ $timeout, $sufficient_timeout);
+ $timeout = $sufficient_timeout;
+ }
}
}
@@ -334,18 +377,16 @@
}
if (%$pending) { # still any requests outstanding? are they expired?
- my $timeout_shrink_factor =
+ my $r =
!$allow_aborting_of_expired || !$self->{total_queries_started} ? 1.0
- : 1 - 0.7 * ( ($self->{total_queries_completed} /
- $self->{total_queries_started}) ** 2 );
- dbg("async: timeout shrink factor: %.2f",
- $timeout_shrink_factor) if $timeout_shrink_factor != 1;
-
+ : $self->{total_queries_completed} / $self->{total_queries_started};
+ my $r2 = $r * $r; # 0..1
while (my($key,$ent) = each %$pending) {
$typecount{$ent->{type}}++;
- my $dt = $ent->{timeout_initial} * $timeout_shrink_factor;
- # don't shrink below a timeout's lower bound
- $dt = $ent->{timeout_min} if $dt < $ent->{timeout_min};
+ my $t_init = $ent->{timeout_initial};
+ my $dt = $t_init - ($t_init - $ent->{timeout_min}) * $r2;
+ # adjust to timer resolution, only deals with 1s and fine resolution
+ $dt = 1 + int $dt if $timer_resolution == 1 && $dt > int $dt;
$allexpired = 0 if $now <= $ent->{start_time} + $dt;
}
dbg("async: queries completed: %d, started: %d",
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?rev=585292&r1=585291&r2=585292&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Tue Oct 16 17:04:48 2007
@@ -2444,25 +2444,35 @@
}
});
-=item rbl_timeout n (default: 15)
+=item rbl_timeout t [t_min] [zone] (default: 15 3)
-All DNS queries are made at the beginning of a check and we try to read the
-results at the end. This value specifies the maximum period of time to wait
-for an DNS query. If most of the DNS queries have succeeded for a particular
-message, then SpamAssassin will not wait for the full period to avoid wasting
-time on unresponsive server(s). For the default 15 second timeout, here is a
-chart of queries remaining versus the effective timeout in seconds:
-
- queries left 100% 90% 80% 70% 60% 50% 40% 30% 20% 10% 0%
- timeout 15 15 14 14 13 11 10 8 5 3 0
-
-In addition, whenever the effective timeout is lowered due to additional query
-results returning, the remaining queries are always given at least one more
-second before timing out, but the wait time will never exceed rbl_timeout.
-
-For example, if 20 queries are made at the beginning of a message check and 16
-queries have returned (leaving 20%), the remaining 4 queries must finish
-within 5 seconds of the beginning of the check or they will be timed out.
+All DNS queries are made at the beginning of a check and we try to read
+the results at the end. This value specifies the maximum period of time
+(in seconds) to wait for an DNS query. If most of the DNS queries have
+succeeded for a particular message, then SpamAssassin will not wait for
+the full period to avoid wasting time on unresponsive server(s), but will
+shrink the timeout according to a percentage of queries already completed.
+As the number of queries remaining approaches 0, the timeout value will
+gradually approach a t_min value, which is an optional second parameter
+and defaults to 0.2 * t. If t is smaller than t_min, the initial timeout
+is set to t_min. Here is a chart of queries remaining versus the timeout
+in seconds, for the default 15 second / 3 second timeout setting:
+
+ queries left 100% 90% 80% 70% 60% 50% 40% 30% 20% 10% 0%
+ timeout 15 14.9 14.5 13.9 13.1 12.0 10.7 9.1 7.3 5.3 3
+
+For example, if 20 queries are made at the beginning of a message check
+and 16 queries have returned (leaving 20%), the remaining 4 queries should
+finish within 7.3 seconds since their query started or they will be timed out.
+Note that timed out queries are only aborted when there is nothing else left
+for SpamAssassin to do - long evaluation of other rules may grant queries
+additional time.
+
+If a parameter 'zone' is specified (it must end with a letter, which
+distinguishes it from other numeric parametrs), then the setting only
+applies to DNS queries against the specified DNS domain (host, domain or
+RBL (sub)zone). Matching is case-insensitive, the actual domain may be a
+subdomain of the specified zone.
=cut
@@ -2470,7 +2480,29 @@
setting => 'rbl_timeout',
is_admin => 1,
default => 15,
- type => $CONF_TYPE_NUMERIC
+ code => sub {
+ my ($self, $key, $value, $line) = @_;
+ unless (defined $value && $value !~ /^$/) {
+ return $MISSING_REQUIRED_VALUE;
+ }
+ local ($1,$2,$3);
+ unless ($value =~ /^ ( [+-]? \d+ (?: \. \d*)? )
+ (?: \s+ ( [+-]? \d+ (?: \. \d*)? ) )?
+ (?: \s+ (\S* [a-zA-Z]) )? $/xs) {
+ return $INVALID_VALUE;
+ }
+ my $zone = $3;
+ if (!defined $zone) { # a global setting
+ $self->{rbl_timeout} = $1+0;
+ $self->{rbl_timeout_min} = $2+0 if defined $2;
+ }
+ else { # per-zone settings
+ $zone =~ s/^\.//; $zone =~ s/\.$//; # strip leading and trailing dot
+ $zone = lc $zone;
+ $self->{by_zone}{$zone}{rbl_timeout} = $1+0;
+ $self->{by_zone}{$zone}{rbl_timeout_min} = $2+0 if defined $2;
+ }
+ }
});
=item util_rb_tld tld1 tld2 ...
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm?rev=585292&r1=585291&r2=585292&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Dns.pm Tue Oct 16 17:04:48 2007
@@ -110,7 +110,7 @@
my $ent = {
key => $key,
- timeout_initial => $self->{conf}->{rbl_timeout},
+ zone => $host, # serves to fetch other per-zone settings
type => "DNSBL-".$type,
sets => [ ], # filled in below
rules => [ ], # filled in below
@@ -159,7 +159,7 @@
my $ent = {
key => $key,
- timeout_initial => $self->{conf}->{rbl_timeout},
+ zone => $host, # serves to fetch other per-zone settings
type => "DNSBL-".$type,
rules => [ $rule ],
# id is filled in after we send the query below
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm?rev=585292&r1=585291&r2=585292&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/ASN.pm Tue Oct 16 17:04:48 2007
@@ -213,15 +213,16 @@
# do the DNS query, have the callback process the result rather than poll for them later
my $zone_index = $index;
+ my $zone = $reversed_ip_quad . '.' . $entry->{zone};
my $key = "asnlookup-${zone_index}-$entry->{zone}";
- my $id = $scanner->{main}->{resolver}->bgsend("${reversed_ip_quad}.$entry->{zone}", 'TXT', undef, sub {
+ my $id = $scanner->{main}->{resolver}->bgsend($zone, 'TXT', undef, sub {
my ($pkt, $id, $timestamp) = @_;
$scanner->{async}->set_response_packet($id, $pkt, $key, $timestamp);
$self->process_dns_result($scanner, $pkt, $zone_index);
});
my $ent = {
key=>$key, id=>$id, type=>'TXT',
- # timeout => ... # defaults to $scanner->{conf}->{rbl_timeout}
+ zone => $zone, # serves to fetch other per-zone settings
};
$scanner->{async}->start_lookup($ent);
dbg("asn: launched DNS TXT query for %s.%s in background",
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm?rev=585292&r1=585291&r2=585292&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/URIDNSBL.pm Tue Oct 16 17:04:48 2007
@@ -471,7 +471,7 @@
return if $scanner->{async}->get_lookup($key);
# dig $dom ns
- my $ent = $self->start_lookup($scanner, 'NS',
+ my $ent = $self->start_lookup($scanner, $dom, 'NS',
$self->res_bgsend($scanner, $dom, 'NS', $key),
$key);
$ent->{obj} = $obj;
@@ -517,7 +517,7 @@
return if $scanner->{async}->get_lookup($key);
# dig $hname a
- my $ent = $self->start_lookup($scanner, 'A',
+ my $ent = $self->start_lookup($scanner, $hname, 'A',
$self->res_bgsend($scanner, $hname, 'A', $key),
$key);
$ent->{obj} = $obj;
@@ -562,7 +562,7 @@
my $item = $lookupstr.".".$dnsbl;
# dig $ip txt
- my $ent = $self->start_lookup($scanner, 'DNSBL',
+ my $ent = $self->start_lookup($scanner, $item, 'DNSBL',
$self->res_bgsend($scanner, $item, $qtype, $key),
$key);
$ent->{obj} = $obj;
@@ -650,11 +650,11 @@
# ---------------------------------------------------------------------------
sub start_lookup {
- my ($self, $scanner, $type, $id, $key) = @_;
+ my ($self, $scanner, $zone, $type, $id, $key) = @_;
my $ent = {
key => $key,
- timeout_initial => $scanner->{conf}->{rbl_timeout},
+ zone => $zone, # serves to fetch other per-zone settings
type => "URI-".$type,
id => $id,
completed_callback => sub {