You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by gb...@apache.org on 2023/07/13 09:48:49 UTC
svn commit: r1910959 - in /spamassassin/trunk: ./ lib/Mail/SpamAssassin/ lib/Mail/SpamAssassin/Plugin/ lib/Mail/SpamAssassin/Pyzor/ lib/Mail/SpamAssassin/Pyzor/Digest/ t/
Author: gbechis
Date: Thu Jul 13 09:48:49 2023
New Revision: 1910959
URL: http://svn.apache.org/viewvc?rev=1910959&view=rev
Log:
Add a Pyzor Perl implementation, disabled by default
Added:
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Client.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm
Modified:
spamassassin/trunk/MANIFEST
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Pyzor.pm
spamassassin/trunk/t/pyzor.t
Modified: spamassassin/trunk/MANIFEST
URL: http://svn.apache.org/viewvc/spamassassin/trunk/MANIFEST?rev=1910959&r1=1910958&r2=1910959&view=diff
==============================================================================
--- spamassassin/trunk/MANIFEST (original)
+++ spamassassin/trunk/MANIFEST Thu Jul 13 09:48:49 2023
@@ -123,6 +123,11 @@ lib/Mail/SpamAssassin/Plugin/WLBLEval.pm
lib/Mail/SpamAssassin/Plugin/WelcomeListSubject.pm
lib/Mail/SpamAssassin/PluginHandler.pm
lib/Mail/SpamAssassin/Plugin/URILocalBL.pm
+lib/Mail/SpamAssassin/Pyzor/Client.pm
+lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm
+lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm
+lib/Mail/SpamAssassin/Pyzor/Digest.pm
+lib/Mail/SpamAssassin/Pyzor.pm
lib/Mail/SpamAssassin/RegistryBoundaries.pm
lib/Mail/SpamAssassin/Reporter.pm
lib/Mail/SpamAssassin/SQLBasedAddrList.pm
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Pyzor.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Pyzor.pm?rev=1910959&r1=1910958&r2=1910959&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Pyzor.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Pyzor.pm Thu Jul 13 09:48:49 2023
@@ -147,7 +147,7 @@ result. Final decision is made by pyzor
Previously pyzor_whitelist_factor which will work interchangeably until 4.1.
-Ignore Pyzor result if REPORTCOUNT x NUMBER >= pyzor_welcomelist_min.
+Ignore Pyzor result if REPORTCOUNT x NUMBER E<gt>= pyzor_welcomelist_min.
For default setting this means: 50 reports requires 10 welcomelistings.
=cut
@@ -180,6 +180,19 @@ throughput. Considered experimental on W
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
});
+=item pyzor_perl (0|1) (default: 0)
+
+Instead of running Pyzor client, use a pure Perl client.
+
+=cut
+
+ push(@cmds, {
+ setting => 'pyzor_perl',
+ is_admin => 1,
+ default => 0,
+ type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
+ });
+
=item pyzor_timeout n (default: 5)
How many seconds you wait for Pyzor to complete, before scanning continues
@@ -227,11 +240,11 @@ characters in the range [0-9A-Za-z =,._/
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value !~ m{^([0-9A-Za-z =,._/-]+)$}) {
- return $Mail::SpamAssassin::Conf::INVALID_VALUE;
+ return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
$self->{pyzor_options} = $1;
}
- });
+});
=item pyzor_path STRING
@@ -250,38 +263,89 @@ you should use this, as the current PATH
code => sub {
my ($self, $key, $value, $line) = @_;
if (!defined $value || !length $value) {
- return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
+ return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
$value = untaint_file_path($value);
if (!-x $value) {
- info("config: pyzor_path \"$value\" isn't an executable");
- return $Mail::SpamAssassin::Conf::INVALID_VALUE;
+ info("config: pyzor_path \"$value\" isn't an executable");
+ return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
$self->{pyzor_path} = $value;
}
});
+=item pyzor_server_file FILE
+
+Pyzor servers configuration file path, used by Pyzor Perl implementation.
+By default Pyzor will connect to public.pyzor.org on port 24441.
+
+=cut
+
+ push (@cmds, {
+ setting => 'pyzor_server_file',
+ is_admin => 1,
+ type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
+ code => sub {
+ my ($self, $key, $value, $line) = @_;
+ if (! -f $value) {
+ return $Mail::SpamAssassin::Conf::INVALID_VALUE;
+ } else {
+ my $i = 0;
+ open(PSF, '<', untaint_file_path($value));
+ while(<PSF>) {
+ chomp;
+ next if (/^#/);
+ my ($host, $port) = split(/:/, $_);
+ $self->{pyzor_host}[$i] = $host;
+ $self->{pyzor_port}[$i] = $port;
+ dbg("Setting Pyzor host to $host and port to $port");
+ $i++;
+ }
+ close(PSF);
+ }
+ }
+ });
+
$conf->{parser}->register_commands(\@cmds);
}
sub is_pyzor_available {
- my ($self) = @_;
+ my ($self) = @_;
- my $pyzor = $self->{main}->{conf}->{pyzor_path} ||
- Mail::SpamAssassin::Util::find_executable_in_env_path('pyzor');
-
- unless ($pyzor && -x $pyzor) {
- dbg("pyzor: no pyzor executable found");
- $self->{pyzor_available} = 0;
- return 0;
- }
-
- # remember any found pyzor
- $self->{main}->{conf}->{pyzor_path} = $pyzor;
+ if($self->{main}->{conf}->{pyzor_perl} and $self->{main}->{conf}->{pyzor_fork}) {
+ info("pyzor: pyzor_perl and pyzor_fork cannot be enabled at the same time, Pyzor support will be disabled");
+ return 0;
+ }
+ local $@;
+ if($self->{main}->{conf}->{pyzor_perl}) {
+ eval {
+ require Mail::SpamAssassin::Pyzor::Digest;
+ require Mail::SpamAssassin::Pyzor::Client;
+ };
+ if($@) {
+ $self->{pyzor_available} = 0;
+ dbg("pyzor: pyzor Perl module is not available");
+ return 0;
+ } else {
+ $self->{pyzor_available} = 1;
+ dbg("pyzor: pyzor Perl module is available");
+ return 1;
+ }
+ } else {
+ my $pyzor = $self->{main}->{conf}->{pyzor_path} ||
+ Mail::SpamAssassin::Util::find_executable_in_env_path('pyzor');
+ unless ($pyzor && -x $pyzor) {
+ dbg("pyzor: no pyzor executable found");
+ $self->{pyzor_available} = 0;
+ return 0;
+ }
- dbg("pyzor: pyzor is available: $pyzor");
- return 1;
+ # remember any found pyzor
+ $self->{main}->{conf}->{pyzor_path} = $pyzor;
+ dbg("pyzor: pyzor is available: $pyzor");
+ return 1;
+ }
}
sub finish_parsing_start {
@@ -314,17 +378,23 @@ sub check_pyzor {
$pms->{tag_data}->{PYZOR} = '';
# create fulltext tmpfile now (before possible forking)
- $pms->{pyzor_tmpfile} = $pms->create_fulltext_tmpfile();
+ if(!$self->{main}->{conf}->{pyzor_perl}) {
+ $pms->{pyzor_tmpfile} = $pms->create_fulltext_tmpfile();
+ }
## non-forking method
-
- if (!$self->{main}->{conf}->{pyzor_fork}) {
+ if ((!$self->{main}->{conf}->{pyzor_fork}) and (!$self->{main}->{conf}->{pyzor_perl})) {
my @results = $self->pyzor_lookup($pms);
return $self->_check_result($pms, \@results);
}
- ## forking method
+ ## Pure Perl implementation
+ if($self->{main}->{conf}->{pyzor_perl}) {
+ my @results = $self->pyzor_lookup($pms, $full);
+ return $self->_check_result($pms, \@results);
+ }
+ ## forking method
$pms->{pyzor_rulename} = $pms->get_current_eval_rule_name();
# create socketpair for communication
@@ -384,76 +454,121 @@ sub check_pyzor {
}
sub pyzor_lookup {
- my ($self, $pms) = @_;
-
+ my ( $self, $pms, $fulltext ) = @_;
my $conf = $self->{main}->{conf};
my $timeout = $conf->{pyzor_timeout};
+ my $server_port;
+ my @pyzor_hosts;
+ my @pyzor_ports;
+ if(defined $conf->{pyzor_host}) {
+ @pyzor_hosts = @{$conf->{pyzor_host}};
+ }
+ if(defined $conf->{pyzor_port}) {
+ @pyzor_ports = @{$conf->{pyzor_port}};
+ }
+ $pyzor_hosts[0] //= 'public.pyzor.org';
+ $pyzor_ports[0] //= 24441;
+
+ if($self->{main}->{conf}->{pyzor_perl}) {
+ my @resp;
+ my $cnt = 0;
+ my $resp;
+ my $ref;
+
+ my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $pms, $fulltext );
+
+ foreach my $server_host ( @pyzor_hosts ) {
+ $server_port = $pyzor_ports[$cnt];
+ $cnt++;
+ my $client = Mail::SpamAssassin::Pyzor::Client->new( 'timeout' => $timeout,
+ 'server_host' => $server_host,
+ 'server_port' => $server_port );
+ local $@;
+ $ref = eval { $client->check($digest); };
+ # $client reply must be an hash
+ return 0 if (not (ref $ref eq ref {}));
+ if ($@) {
+ my $err = $@;
- # note: not really tainted, this came from system configuration file
- my $path = untaint_file_path($conf->{pyzor_path});
- my $opts = untaint_var($conf->{pyzor_options}) || '';
-
- $pms->enter_helper_run_mode();
-
- my $pid;
- my @resp;
- my $timer = Mail::SpamAssassin::Timeout->new(
- { secs => $timeout, deadline => $pms->{master_deadline} });
- my $err = $timer->run_and_catch(sub {
- local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" };
-
- dbg("pyzor: opening pipe: ".
- join(' ', $path, $opts, "check", "<".$pms->{pyzor_tmpfile}));
-
- $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR,
- $pms->{pyzor_tmpfile}, 1, $path, split(' ', $opts), "check");
- $pid or die "$!\n";
-
- # read+split avoids a Perl I/O bug (Bug 5985)
- my($inbuf, $nread);
- my $resp = '';
- while ($nread = read(PYZOR, $inbuf, 8192)) { $resp .= $inbuf }
- defined $nread or die "error reading from pipe: $!";
- @resp = split(/^/m, $resp, -1);
+ $err = eval { $err->get_message() } || $err;
- my $errno = 0;
- close PYZOR or $errno = $!;
- if (proc_status_ok($?, $errno)) {
- dbg("pyzor: [%s] finished successfully", $pid);
- } elsif (proc_status_ok($?, $errno, 0, 1)) { # sometimes it exits with 1
- dbg("pyzor: [%s] finished: %s", $pid, exit_status_str($?, $errno));
- } else {
- info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno));
+ warn("pyzor: check failed: $err\n");
+ return 0;
+ }
+ # mimic pyzor client reply
+ # public.pyzor.org:24441\t(200, 'OK')\t1\t0
+ $resp .= "$server_host:$server_port\t($ref->{'Code'}, '" . $ref->{'Diag'} . "')\t$ref->{'Count'}\t$ref->{'WL-Count'}\n";
+ undef $ref;
}
+ @resp = split(/^/m, $resp, -1);
+ return @resp;
+ } else {
+ # note: not really tainted, this came from system configuration file
+ my $path = untaint_file_path($conf->{pyzor_path});
+ my $opts = untaint_var($conf->{pyzor_options}) || '';
+
+ $pms->enter_helper_run_mode();
+
+ my $pid;
+ my @resp;
+ my $timer = Mail::SpamAssassin::Timeout->new(
+ { secs => $timeout, deadline => $pms->{master_deadline} });
+ my $err = $timer->run_and_catch(sub {
+ local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" };
+
+ dbg("pyzor: opening pipe: ".
+ join(' ', $path, $opts, "check", "<".$pms->{pyzor_tmpfile}));
+
+ $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR,
+ $pms->{pyzor_tmpfile}, 1, $path, split(' ', $opts), "check");
+ $pid or die "$!\n";
+
+ # read+split avoids a Perl I/O bug (Bug 5985)
+ my($inbuf, $nread);
+ my $resp = '';
+ while ($nread = read(PYZOR, $inbuf, 8192)) { $resp .= $inbuf }
+ defined $nread or die "error reading from pipe: $!";
+ @resp = split(/^/m, $resp, -1);
+
+ my $errno = 0;
+ close PYZOR or $errno = $!;
+ if (proc_status_ok($?, $errno)) {
+ dbg("pyzor: [%s] finished successfully", $pid);
+ } elsif (proc_status_ok($?, $errno, 0, 1)) { # sometimes it exits with 1
+ dbg("pyzor: [%s] finished: %s", $pid, exit_status_str($?, $errno));
+ } else {
+ info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno));
+ }
- });
+ });
- if (defined(fileno(*PYZOR))) { # still open
- if ($pid) {
- if (kill('TERM', $pid)) {
- dbg("pyzor: killed stale helper [$pid]");
- } else {
- dbg("pyzor: killing helper application [$pid] failed: $!");
+ if (defined(fileno(*PYZOR))) { # still open
+ if ($pid) {
+ if (kill('TERM', $pid)) {
+ dbg("pyzor: killed stale helper [$pid]");
+ } else {
+ dbg("pyzor: killing helper application [$pid] failed: $!");
+ }
}
+ my $errno = 0;
+ close PYZOR or $errno = $!;
+ proc_status_ok($?, $errno)
+ or info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno));
}
- my $errno = 0;
- close PYZOR or $errno = $!;
- proc_status_ok($?, $errno)
- or info("pyzor: [%s] error: %s", $pid, exit_status_str($?, $errno));
- }
- $pms->leave_helper_run_mode();
+ $pms->leave_helper_run_mode();
- if ($timer->timed_out()) {
- dbg("pyzor: check timed out after $timeout seconds");
- return ();
- } elsif ($err) {
- chomp $err;
- info("pyzor: check failed: $err");
- return ();
- }
+ if ($timer->timed_out()) {
+ dbg("pyzor: check timed out after $timeout seconds");
+ return ();
+ } elsif ($err) {
+ chomp $err;
+ info("pyzor: check failed: $err");
+ return ();
+ }
- return @resp;
+ return @resp;
+ }
}
sub check_tick {
@@ -564,7 +679,7 @@ sub _check_result {
my $wl_min = $conf->{pyzor_welcomelist_min};
my $wl_limit = $count_wl >= $wl_min ?
- $count * $conf->{pyzor_welcomelist_factor} : 0;
+ $count * $conf->{pyzor_welcomelist_factor} : 0;
dbg("pyzor: result: COUNT=$count/$count_min WELCOMELIST=$count_wl/$wl_min/%.1f",
$wl_limit);
@@ -601,84 +716,138 @@ sub plugin_report {
return if $options->{report}->{options}->{dont_report_to_pyzor};
return if !$self->is_pyzor_available();
- # use temporary file: open2() is unreliable due to buffering under spamd
- my $tmpf = $options->{report}->create_fulltext_tmpfile($options->{text});
- if ($self->pyzor_report($options, $tmpf)) {
- $options->{report}->{report_available} = 1;
- info("reporter: spam reported to Pyzor");
- $options->{report}->{report_return} = 1;
- }
- else {
- info("reporter: could not report spam to Pyzor");
+ if($self->{main}->{conf}->{pyzor_perl}) {
+ if (!$options->{report}->{options}->{dont_report_to_pyzor} && $self->is_pyzor_available()) {
+ if ($self->pyzor_report($options)) {
+ $options->{report}->{report_available} = 1;
+ info("reporter: spam reported to Pyzor");
+ $options->{report}->{report_return} = 1;
+ } else {
+ info("reporter: could not report spam to Pyzor");
+ }
+ }
+ } else {
+ # use temporary file: open2() is unreliable due to buffering under spamd
+ my $tmpf = $options->{report}->create_fulltext_tmpfile($options->{text});
+ if ($self->pyzor_report($options, $tmpf)) {
+ $options->{report}->{report_available} = 1;
+ info("reporter: spam reported to Pyzor");
+ $options->{report}->{report_return} = 1;
+ }
+ else {
+ info("reporter: could not report spam to Pyzor");
+ }
+ $options->{report}->delete_fulltext_tmpfile($tmpf);
}
- $options->{report}->delete_fulltext_tmpfile($tmpf);
-
return 1;
}
sub pyzor_report {
- my ($self, $options, $tmpf) = @_;
+ my ( $self, $options, $tmpf ) = @_;
- # note: not really tainted, this came from system configuration file
- my $path = untaint_file_path($options->{report}->{conf}->{pyzor_path});
- my $opts = untaint_var($options->{report}->{conf}->{pyzor_options}) || '';
+ my $conf = $self->{main}->{conf};
+
+ my $server_port;
+ my @pyzor_hosts;
+ my @pyzor_ports;
+ if(defined $conf->{pyzor_host}) {
+ @pyzor_hosts = @{$conf->{pyzor_host}};
+ }
+ if(defined $conf->{pyzor_port}) {
+ @pyzor_ports = @{$conf->{pyzor_port}};
+ }
+ $pyzor_hosts[0] //= 'public.pyzor.org';
+ $pyzor_ports[0] //= 24441;
my $timeout = $self->{main}->{conf}->{pyzor_timeout};
- $options->{report}->enter_helper_run_mode();
+ if($self->{main}->{conf}->{pyzor_perl}) {
+ my $pms =
+ Mail::SpamAssassin::PerMsgStatus->new($self->{main}, $options->{msg});
+
+ my $cnt = 0;
+ foreach my $server_host ( @pyzor_hosts ) {
+ $server_port = $pyzor_ports[$cnt];
+ $cnt++;
+ my $client = Mail::SpamAssassin::Pyzor::Client->new( 'timeout' => $timeout,
+ 'server_host' => $server_host,
+ 'server_port' => $server_port );
+
+ my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $pms );
+
+ local $@;
+ my $ref = eval { $client->report($digest); };
+ if ($@) {
+ warn("pyzor: report failed: $@");
+ return 0;
+ }
+ elsif ( $ref->{'Code'} ne 200 ) {
+ dbg("pyzor: report failed with invalid code: $ref->{'Code'}: $ref->{'Diag'}");
+ return 0;
+ }
+ }
+ } else {
+ # note: not really tainted, this came from system configuration file
+ my $path = untaint_file_path($options->{report}->{conf}->{pyzor_path});
+ my $opts = untaint_var($options->{report}->{conf}->{pyzor_options}) || '';
- my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout });
- my $err = $timer->run_and_catch(sub {
+ my $timeout = $self->{main}->{conf}->{pyzor_timeout};
- local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" };
+ $options->{report}->enter_helper_run_mode();
- dbg("pyzor: opening pipe: " . join(' ', $path, $opts, "report", "< $tmpf"));
+ my $timer = Mail::SpamAssassin::Timeout->new({ secs => $timeout });
+ my $err = $timer->run_and_catch(sub {
- my $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR,
- $tmpf, 1, $path, split(' ', $opts), "report");
- $pid or die "$!\n";
+ local $SIG{PIPE} = sub { die "__brokenpipe__ignore__\n" };
- my($inbuf,$nread,$nread_all); $nread_all = 0;
- # response is ignored, just check its existence
- while ( $nread=read(PYZOR,$inbuf,8192) ) { $nread_all += $nread }
- defined $nread or die "error reading from pipe: $!";
+ dbg("pyzor: opening pipe: " . join(' ', $path, $opts, "report", "< $tmpf"));
- dbg("pyzor: empty response") if $nread_all < 1;
+ my $pid = Mail::SpamAssassin::Util::helper_app_pipe_open(*PYZOR,
+ $tmpf, 1, $path, split(' ', $opts), "report");
+ $pid or die "$!\n";
- my $errno = 0; close PYZOR or $errno = $!;
- # closing a pipe also waits for the process executing on the pipe to
- # complete, no need to explicitly call waitpid
- # my $child_stat = waitpid($pid,0) > 0 ? $? : undef;
- if (proc_status_ok($?,$errno, 0)) {
- dbg("pyzor: [%s] reporter finished successfully", $pid);
- } else {
- info("pyzor: [%s] reporter error: %s", $pid, exit_status_str($?,$errno));
- }
+ my($inbuf,$nread,$nread_all); $nread_all = 0;
+ # response is ignored, just check its existence
+ while ( $nread=read(PYZOR,$inbuf,8192) ) { $nread_all += $nread }
+ defined $nread or die "error reading from pipe: $!";
- });
+ dbg("pyzor: empty response") if $nread_all < 1;
- $options->{report}->leave_helper_run_mode();
+ my $errno = 0; close PYZOR or $errno = $!;
+ # closing a pipe also waits for the process executing on the pipe to
+ # complete, no need to explicitly call waitpid
+ # my $child_stat = waitpid($pid,0) > 0 ? $? : undef;
+ if (proc_status_ok($?,$errno, 0)) {
+ dbg("pyzor: [%s] reporter finished successfully", $pid);
+ } else {
+ info("pyzor: [%s] reporter error: %s", $pid, exit_status_str($?,$errno));
+ }
- if ($timer->timed_out()) {
- dbg("reporter: pyzor report timed out after $timeout seconds");
- return 0;
- }
+ });
- if ($err) {
- chomp $err;
- if ($err eq '__brokenpipe__ignore__') {
- dbg("reporter: pyzor report failed: broken pipe");
- } else {
- warn("reporter: pyzor report failed: $err\n");
+ $options->{report}->leave_helper_run_mode();
+
+ if ($timer->timed_out()) {
+ dbg("reporter: pyzor report timed out after $timeout seconds");
+ return 0;
}
- return 0;
- }
+ if ($err) {
+ chomp $err;
+ if ($err eq '__brokenpipe__ignore__') {
+ dbg("reporter: pyzor report failed: broken pipe");
+ } else {
+ warn("reporter: pyzor report failed: $err\n");
+ }
+ return 0;
+ }
+ }
return 1;
}
# Version features
sub has_fork { 1 }
+sub has_perl { 1 }
1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor.pm?rev=1910959&view=auto
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor.pm (added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor.pm Thu Jul 13 09:48:49 2023
@@ -0,0 +1,56 @@
+package Mail::SpamAssassin::Pyzor;
+
+# Copyright 2018 cPanel, LLC.
+# All rights reserved.
+# http://cpanel.net
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+#
+
+use strict;
+use warnings;
+
+our $VERSION = '0.06_01';
+
+=encoding utf-8
+
+=head1 NAME
+
+Mail::SpamAssassin::Pyzor - Pyzor spam filtering in Perl
+
+=head1 DESCRIPTION
+
+This distribution contains Perl implementations of parts of
+L<Pyzor|http://pyzor.org>, a tool for use in spam email filtering.
+It is intended for use with L<Mail::SpamAssassin> but may be useful
+in other contexts.
+
+See the following modules for information on specific tools that
+the distribution includes:
+
+=over
+
+=item * L<Mail::SpamAssassin::Pyzor::Client>
+
+=item * L<Mail::SpamAssassin::Pyzor::Digest>
+
+=back
+
+=cut
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Client.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Client.pm?rev=1910959&view=auto
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Client.pm (added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Client.pm Thu Jul 13 09:48:49 2023
@@ -0,0 +1,419 @@
+package Mail::SpamAssassin::Pyzor::Client;
+
+# Copyright 2018 cPanel, LLC.
+# All rights reserved.
+# http://cpanel.net
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+#
+
+use strict;
+use warnings;
+
+=encoding utf-8
+
+=head1 NAME
+
+Mail::SpamAssassin::Pyzor::Client - Pyzor client logic
+
+=head1 SYNOPSIS
+
+ use Mail::SpamAssassin::Pyzor::Client ();
+ use Mail::SpamAssassin::Pyzor::Digest ();
+
+ my $client = Mail::SpamAssassin::Pyzor::Client->new();
+
+ my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $msg );
+
+ my $check_ref = $client->check($digest);
+ die $check_ref->{'Diag'} if $check_ref->{'Code'} ne '200';
+
+ my $report_ref = $client->report($digest);
+ die $report_ref->{'Diag'} if $report_ref->{'Code'} ne '200';
+
+=head1 DESCRIPTION
+
+A bare-bones L<Pyzor|http://pyzor.org> client that currently only
+implements the functionality needed for L<Mail::SpamAssassin>.
+
+=head1 PROTOCOL DETAILS
+
+The Pyzor protocol is not a published standard, and there appears to be
+no meaningful public documentation. What follows is enough information,
+largely gleaned through forum posts and reverse engineering, to facilitate
+effective use of this module:
+
+Pyzor is an RPC-oriented, message-based protocol. Each message
+is a simple dictionary of 7-bit ASCII keys and values. Server responses
+always include at least the following:
+
+=over
+
+=item * C<Code> - Similar to HTTP status codes; anything besides C<200>
+is an error.
+
+=item * C<Diag> - Similar to HTTP status reasons: a text description
+of the status.
+
+=back
+
+(NB: There are additional standard response headers that are useful only for
+the protocol itself and thus are not part of this moduleâs returns.)
+
+=head2 Reliability
+
+Pyzor uses UDP rather than TCP, so no message is guaranteed to reach its
+destination. A transmission failure can happen in either the request or
+the response; in either case, a timeout error will result. Such errors
+are represented as thrown instances of L<Mail::Pyzor::X::Timeout>.
+
+=cut
+
+#----------------------------------------------------------------------
+
+our $VERSION = '0.04';
+
+our $DEFAULT_SERVER_HOST = 'public.pyzor.org';
+our $DEFAULT_SERVER_PORT = 24441;
+our $DEFAULT_USERNAME = 'anonymous';
+our $DEFAULT_PASSWORD = '';
+our $DEFAULT_OP_SPEC = '20,3,60,3';
+our $PYZOR_PROTOCOL_VERSION = 2.1;
+our $DEFAULT_TIMEOUT = 3.5;
+our $READ_SIZE = 8192;
+
+use IO::Socket::INET ();
+use Digest::SHA qw(sha1 sha1_hex);
+use Mail::SpamAssassin::Util qw(untaint_var);
+
+my @hash_order = ( 'Op', 'Op-Digest', 'Op-Spec', 'Thread', 'PV', 'User', 'Time', 'Sig' );
+
+#----------------------------------------------------------------------
+
+=head1 CONSTRUCTOR
+
+=head2 new(%OPTS)
+
+Create a new pyzor client.
+
+=over 2
+
+=item Input
+
+%OPTS are (all optional):
+
+=over 3
+
+=item * C<server_host> - The pyzor server host to connect to (default is
+C<public.pyzor.org>)
+
+=item * C<server_port> - The pyzor server port to connect to (default is
+24441)
+
+=item * C<username> - The username to present to the pyzor server (default
+is C<anonymous>)
+
+=item * C<password> - The password to present to the pyzor server (default
+is empty)
+
+=item * C<timeout> - The maximum time, in seconds, to wait for a response
+from the pyzor server (defeault is 3.5)
+
+=back
+
+=item Output
+
+=over 3
+
+Returns a L<Mail::SpamAssassin::Pyzor::Client> object.
+
+=back
+
+=back
+
+=cut
+
+sub new {
+ my ( $class, %OPTS ) = @_;
+
+ $OPTS{'server_host'} = untaint_var($OPTS{'server_host'});
+ $OPTS{'server_port'} = untaint_var($OPTS{'server_port'});
+ $OPTS{'username'} = untaint_var($OPTS{'username'});
+ $OPTS{'password'} = untaint_var($OPTS{'password'});
+ $OPTS{'timeout'} = untaint_var($OPTS{'timeout'});
+
+ return bless {
+ 'server_host' => $OPTS{'server_host'} || $DEFAULT_SERVER_HOST,
+ 'server_port' => $OPTS{'server_port'} || $DEFAULT_SERVER_PORT,
+ 'username' => $OPTS{'username'} || $DEFAULT_USERNAME,
+ 'password' => $OPTS{'password'} || $DEFAULT_PASSWORD,
+ 'op_spec' => $DEFAULT_OP_SPEC,
+ 'timeout' => $OPTS{'timeout'} || $DEFAULT_TIMEOUT,
+ }, $class;
+}
+
+#----------------------------------------------------------------------
+
+=head1 REQUEST METHODS
+
+=head2 report($digest)
+
+Report the digest of a spam message to the pyzor server. This function
+will throw if a messaging failure or timeout happens.
+
+=over 2
+
+=item Input
+
+=over 3
+
+=item $digest C<SCALAR>
+
+The message digest to report, as given by
+C<Mail::SpamAssassin::Pyzor::Digest::get()>.
+
+=back
+
+=item Output
+
+=over 3
+
+=item C<HASHREF>
+
+Returns a hashref of the standard attributes noted above.
+
+=back
+
+=back
+
+=cut
+
+sub report {
+ my ( $self, $digest ) = @_;
+
+ my $msg_ref = $self->_get_base_msg( 'report', $digest );
+
+ $msg_ref->{'Op-Spec'} = $self->{'op_spec'};
+
+ return $self->_send_receive_msg($msg_ref);
+}
+
+=head2 check($digest)
+
+Check the digest of a message to see if
+the pyzor server has a report for it. This function
+will throw if a messaging failure or timeout happens.
+
+=over 2
+
+=item Input
+
+=over 3
+
+=item $digest C<SCALAR>
+
+The message digest to check, as given by
+C<Mail::SpamAssassin::Pyzor::Digest::get()>.
+
+=back
+
+=item Output
+
+=over 3
+
+=item C<HASHREF>
+
+Returns a hashref of the standard attributes noted above
+as well as the following:
+
+=over
+
+=item * C<Count> - The number of reports the server has received
+for the given digest.
+
+=item * C<WL-Count> - The number of whitelist requests the server has received
+for the given digest.
+
+=back
+
+=back
+
+=back
+
+=cut
+
+sub check {
+ my ( $self, $digest ) = @_;
+
+ return $self->_send_receive_msg( $self->_get_base_msg( 'check', $digest ) );
+}
+
+# ----------------------------------------
+
+sub _send_receive_msg {
+ my ( $self, $msg_ref ) = @_;
+
+ my $thread_id = $msg_ref->{'Thread'} or warn 'No thread ID?';
+
+ $self->_sign_msg($msg_ref);
+
+ return $self->_do_send_receive(
+ $self->_generate_packet_from_message($msg_ref) . "\n\n",
+ $thread_id,
+ );
+}
+
+sub _get_base_msg {
+ my ( $self, $op, $digest ) = @_;
+
+ die "Implementor error: op is required" if !$op;
+ die "error: digest is required" if !$digest;
+
+ return {
+ 'User' => $self->{'_username'},
+ 'PV' => $PYZOR_PROTOCOL_VERSION,
+ 'Time' => time(),
+ 'Op' => $op,
+ 'Op-Digest' => $digest,
+ 'Thread' => $self->_generate_thread_id()
+ };
+}
+
+sub _do_send_receive {
+ my ( $self, $packet, $thread_id ) = @_;
+
+ my $sock = $self->_get_connection_or_die();
+
+ $self->_send_packet( $sock, $packet );
+ my $response = $self->_receive_packet( $sock, $thread_id );
+
+ return 0 if not defined $response;
+
+ my $resp_hr = { map { ( split(m{: }) )[ 0, 1 ] } split( m{\n}, $response ) };
+
+ delete $resp_hr->{'Thread'};
+
+ my $response_pv = delete $resp_hr->{'PV'};
+
+ if ( $PYZOR_PROTOCOL_VERSION ne $response_pv ) {
+ warn "Unexpected protocol version ($response_pv) in Pyzor response!";
+ }
+
+ return $resp_hr;
+}
+
+sub _receive_packet {
+ my ( $self, $sock, $thread_id ) = @_;
+
+ my $timeout = $self->{'timeout'} * 1000;
+
+ my $end_time = time + $self->{'timeout'};
+
+ $sock->blocking(0);
+ my $response = '';
+ my $rout = '';
+ my $rin = '';
+ vec( $rin, fileno($sock), 1 ) = 1;
+
+ while (1) {
+ my $time_left = $end_time - time;
+
+ if ( $time_left <= 0 ) {
+ warn("Did not receive a response from the pyzor server $self->{'server_host'}:$self->{'server_port'} for $self->{'timeout'} seconds!");
+ return;
+ }
+
+ my $bytes = sysread( $sock, $response, $READ_SIZE, length $response );
+ if ( !defined($bytes) && !$!{'EAGAIN'} && !$!{'EWOULDBLOCK'} ) {
+ warn "read from socket: $!";
+ }
+
+ if ( index( $response, "\n\n" ) > -1 ) {
+
+ # Reject the response unless its thread ID matches what we sent.
+ # This prevents confusion among concurrent Pyzor requests.
+ if ( index( $response, "\nThread: $thread_id\n" ) != -1 ) {
+ last;
+ }
+ else {
+ $response = '';
+ }
+ }
+
+ my $found = select( $rout = $rin, undef, undef, $time_left );
+ warn "select(): $!" if $found == -1;
+ }
+
+ return $response;
+}
+
+sub _send_packet {
+ my ( $self, $sock, $packet ) = @_;
+
+ $sock->blocking(1);
+ syswrite( $sock, $packet ) or warn "write to socket: $!";
+
+ return;
+}
+
+sub _get_connection_or_die {
+ my ($self) = @_;
+
+ # clear the socket
+ undef $self->{'_sock_pid'};
+ undef $self->{'_sock'};
+
+ $self->{'_sock_pid'} ||= $$;
+ $self->{'_sock'} ||= IO::Socket::INET->new(
+ 'PeerHost' => $self->{'server_host'},
+ 'PeerPort' => $self->{'server_port'},
+ 'Proto' => 'udp'
+ ) or die "Cannot connect to $self->{'server_host'}:$self->{'server_port'}: $@ $!";
+ return $self->{'_sock'};
+}
+
+sub _sign_msg {
+ my ( $self, $msg_ref ) = @_;
+
+ $msg_ref->{'Sig'} = lc Digest::SHA::sha1_hex(
+ Digest::SHA::sha1( $self->_generate_packet_from_message($msg_ref) )
+ );
+
+ return 1;
+}
+
+sub _generate_packet_from_message {
+ my ( $self, $msg_ref ) = @_;
+
+ return join( "\n", map { "$_: $msg_ref->{$_}" } grep { length $msg_ref->{$_} } @hash_order );
+}
+
+sub _generate_thread_id {
+ my $RAND_MAX = 2**16;
+ my $val = 0;
+ $val = int rand($RAND_MAX) while $val < 1024;
+ return $val;
+}
+
+sub _get_user_pass_hash_key {
+ my ($self) = @_;
+
+ return lc Digest::SHA::sha1_hex( $self->{'username'} . ':' . $self->{'password'} );
+}
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest.pm?rev=1910959&view=auto
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest.pm (added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest.pm Thu Jul 13 09:48:49 2023
@@ -0,0 +1,99 @@
+package Mail::SpamAssassin::Pyzor::Digest;
+
+# Copyright 2018 cPanel, LLC.
+# All rights reserved.
+# http://cpanel.net
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+#
+
+use strict;
+use warnings;
+
+=encoding utf-8
+
+=head1 NAME
+
+Mail::SpamAssassin::Pyzor::Digest
+
+=head1 SYNOPSIS
+
+ my $digest = Mail::SpamAssassin::Pyzor::Digest::get( $mime_text );
+
+=head1 DESCRIPTION
+
+A reimplementation of L<https://github.com/SpamExperts/pyzor/blob/master/pyzor/digest.py>.
+
+=cut
+
+#----------------------------------------------------------------------
+
+use Mail::SpamAssassin::Pyzor::Digest::Pieces ();
+use Digest::SHA qw(sha1_hex);
+
+our $VERSION = '0.03';
+
+#----------------------------------------------------------------------
+
+=head1 FUNCTIONS
+
+=head2 $hex = get( $MSG )
+
+This takes an email message in raw MIME text format (i.e., as saved in the
+standard mbox format) and returns the messageâs Pyzor digest in lower-case
+hexadecimal.
+
+The output from this function should normally be identical to that of
+the C<pyzor> scriptâs C<digest> command. It is suitable for use in
+L<Mail::SpamAssassin::Pyzor::Client>âs request methods.
+
+=cut
+
+sub get {
+ my ($pms) = @_;
+ return Digest::SHA::sha1_hex( ${ _get_predigest( $pms ) } );
+}
+
+# NB: This is called from the test.
+sub _get_predigest { ## no critic qw(RequireArgUnpacking)
+ my ($pms) = @_;
+
+ my $parsed = $pms->get_message();
+
+ my @lines;
+
+ my $payloads_ar = Mail::SpamAssassin::Pyzor::Digest::Pieces::digest_payloads($parsed);
+ for my $payload (@$payloads_ar) {
+ my @p_lines = Mail::SpamAssassin::Pyzor::Digest::Pieces::splitlines($payload);
+ for my $line (@p_lines) {
+ Mail::SpamAssassin::Pyzor::Digest::Pieces::normalize($line);
+
+ next if !Mail::SpamAssassin::Pyzor::Digest::Pieces::should_handle_line($line);
+
+ # Make sure we have an octet string.
+ utf8::encode($line) if utf8::is_utf8($line);
+
+ push @lines, $line;
+ }
+ }
+
+ my $digest_sr = Mail::SpamAssassin::Pyzor::Digest::Pieces::assemble_lines( \@lines );
+ return $digest_sr;
+}
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm?rev=1910959&view=auto
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm (added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/Pieces.pm Thu Jul 13 09:48:49 2023
@@ -0,0 +1,311 @@
+package Mail::SpamAssassin::Pyzor::Digest::Pieces;
+
+# Copyright 2018 cPanel, LLC.
+# All rights reserved.
+# http://cpanel.net
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+#
+
+use strict;
+use warnings;
+
+=encoding utf-8
+
+=head1 NAME
+
+Mail::SpamAssassin::Pyzor::Digest::Pieces
+
+=head1 DESCRIPTION
+
+This module houses backend logic for L<Mail::SpamAssassin::Pyzor::Digest>.
+
+It reimplements logic found in pyzorâs F<digest.py> module
+(L<https://github.com/SpamExperts/pyzor/blob/master/pyzor/digest.py>).
+
+=cut
+
+#----------------------------------------------------------------------
+
+use Encode ();
+
+our $VERSION = '0.03';
+
+# each tuple is [ offset, length ]
+use constant _HASH_SPEC => ( [ 20, 3 ], [ 60, 3 ] );
+
+use constant {
+ _MIN_LINE_LENGTH => 8,
+
+ _ATOMIC_NUM_LINES => 4,
+};
+
+#----------------------------------------------------------------------
+
+=head1 FUNCTIONS
+
+=head2 $strings_ar = digest_payloads( $EMAIL_MIME )
+
+This imitates the corresponding object method in F<digest.py>.
+It returns a reference to an array of strings. Each string can be either
+a byte string or a character string (e.g., UTF-8 decoded).
+
+NB: RFC 2822 stipulates that message bodies should use CRLF
+line breaks, not plain LF (nor plain CR).
+We will thus convert any plain CRs in a quoted-printable message
+body into CRLF. Python, though, doesnât do this, so the output of
+our implementation of C<digest_payloads()> diverges from that of the Python
+original. It doesnât ultimately make a difference since the line-ending
+whitespace gets trimmed regardless, but itâs necessary to factor in when
+comparing the output of our implementation with the Python output.
+
+=cut
+
+sub digest_payloads {
+ my ($parsed) = @_;
+
+ my @subparts;
+
+ foreach my $part ($parsed->find_parts(qr/./, 1)) {
+ push(@subparts, $part);
+ }
+ my @payloads;
+
+ foreach my $p (@subparts) {
+ my ( $main_type, $subtype, $encoding, $encode_check ) = parse_content_type( $p->{'type'} );
+
+ my $payload;
+
+ if ( $main_type eq 'text' ) {
+
+ if ( $subtype eq 'plain' ) {
+ $payload = $p->{'decoded'};
+ $payload =~ s/\\'/\'/gx;
+ } else {
+ $payload = $p->{'rendered'};
+ }
+
+ utf8::upgrade($payload);
+
+ if ( $subtype eq 'html' ) {
+ require Mail::SpamAssassin::Pyzor::Digest::StripHtml;
+ $payload = Mail::SpamAssassin::Pyzor::Digest::StripHtml::strip($payload);
+ }
+ }
+ else {
+ # This does no decoding, even of, e.g., quoted-printable or base64.
+ $payload = $p->{'pristine_body'};
+ }
+ next if not defined $payload;
+ push @payloads, $payload;
+ }
+ return \@payloads;
+}
+
+#----------------------------------------------------------------------
+
+=head2 normalize( $STRING )
+
+This imitates the corresponding object method in F<digest.py>.
+It modifies C<$STRING> in-place.
+
+As with the original implementation, if C<$STRING> contains (decoded)
+Unicode characters, those characters will be parsed accordingly. So:
+
+ $str = "123\xc2\xa0"; # [ c2 a0 ] == \u00a0, non-breaking space
+
+ normalize($str);
+
+The above will leave C<$str> alone, but this:
+
+ utf8::decode($str);
+
+ normalize($str);
+
+â¦Â will trim off the last two bytes from C<$str>.
+
+=cut
+
+sub normalize { ## no critic qw( Subroutines::RequireArgUnpacking )
+
+ # NULs are bad, mm-kay?
+ $_[0] =~ tr<\0><>d;
+
+ # NB: Pythonâs \s without re.UNICODE is the same as Perlâs \s
+ # with the /a modifier.
+ #
+ # https://docs.python.org/2/library/re.html
+ # https://perldoc.perl.org/perlrecharclass.html#Backslash-sequences
+
+ # Python: re.compile(r'\S{10,}')
+ $_[0] =~ s<\S{10,}><>ag;
+
+ # Python: re.compile(r'\S+@\S+')
+ $_[0] =~ s<\S+ @ \S+><>agx;
+
+ # Python: re.compile(r'[a-z]+:\S+', re.IGNORECASE)
+ $_[0] =~ s<[a-zA-Z]+ : \S+><>agx;
+
+ # (from digest.py â¦)
+ # Make sure we do the whitespace last because some of the previous
+ # patterns rely on whitespace.
+ $_[0] =~ tr< \x09-\x0d><>d;
+
+ # This is fun. digest.pyâs normalize() does a non-UNICODE whitespace
+ # strip, then calls strip() on the string, which *will* strip Unicode
+ # whitespace from the ends.
+ $_[0] =~ s<\A\s+><>;
+ $_[0] =~ s<\s+\z><>;
+
+ return;
+}
+
+#----------------------------------------------------------------------
+
+=head2 $yn = should_handle_line( $STRING )
+
+This imitates the corresponding object method in F<digest.py>.
+It returns a boolean.
+
+=cut
+
+sub should_handle_line {
+ return $_[0] && length( $_[0] ) >= _MIN_LINE_LENGTH();
+}
+
+#----------------------------------------------------------------------
+
+=head2 $sr = assemble_lines( \@LINES )
+
+This assembles a string buffer out of @LINES. The string is the buffer
+of octets that will be hashed to produce the message digest.
+
+Each member of @LINES is expected to be an B<octet string>, not a
+character string.
+
+=cut
+
+sub assemble_lines {
+ my ($lines_ar) = @_;
+
+ if ( @$lines_ar <= _ATOMIC_NUM_LINES() ) {
+
+ # cf. handle_atomic() in digest.py
+ return \join( q<>, @$lines_ar );
+ }
+
+ #----------------------------------------------------------------------
+ # cf. handle_atomic() in digest.py
+
+ my $str = q<>;
+
+ for my $ofs_len ( _HASH_SPEC() ) {
+ my ( $offset, $length ) = @$ofs_len;
+
+ for my $i ( 0 .. ( $length - 1 ) ) {
+ my $idx = int( $offset * @$lines_ar / 100 ) + $i;
+
+ next if !defined $lines_ar->[$idx];
+
+ $str .= $lines_ar->[$idx];
+ }
+ }
+
+ return \$str;
+}
+
+#----------------------------------------------------------------------
+
+=head2 ($main, $sub, $encoding, $checkval) = parse_content_type( $CONTENT_TYPE )
+
+=cut
+
+use constant _QUOTED_PRINTABLE_NAMES => (
+ "quopri-codec",
+ "quopri",
+ "quoted-printable",
+ "quotedprintable",
+);
+
+# Make Encode::decode() ignore anything that doesnât fit the
+# given encoding.
+use constant _encode_check_ignore => q<>;
+
+sub parse_content_type {
+ my ($content_type) = @_;
+
+ # text/plain; charset=us-ascii
+ my $ct_parse;
+ if($content_type =~ /(\w+)\/(\w+); charset=(.*)/) {
+ $ct_parse->{type} = $1;
+ $ct_parse->{subtype} = $2;
+ $ct_parse->{'attributes'}{'charset'} = $3;
+ } elsif($content_type =~ /(\w+)\/(\w+)/) {
+ $ct_parse->{type} = $1;
+ $ct_parse->{subtype} = $2;
+ $ct_parse->{'attributes'}{'charset'} = 'us-ascii';
+ } else {
+ $ct_parse->{type} = 'text';
+ $ct_parse->{subtype} = 'plain';
+ $ct_parse->{'attributes'}{'charset'} = 'us-ascii';
+ }
+
+ my $main = $ct_parse->{'type'} || q<>;
+ my $sub = $ct_parse->{'subtype'} || q<>;
+
+ my $encoding = $ct_parse->{'attributes'}{'charset'};
+
+ my $checkval;
+
+ if ($encoding) {
+
+ # Lower-case everything, convert underscore to dash, and remove NUL.
+ $encoding =~ tr<A-Z_\0><a-z->d;
+
+ # Apparently pyzor accommodates messages that put the transfer
+ # encoding in the Content-Type.
+ if ( grep { $_ eq $encoding } _QUOTED_PRINTABLE_NAMES() ) {
+ $checkval = Encode::FB_CROAK();
+ }
+ }
+ else {
+ $encoding = 'ascii';
+ }
+
+ # Match Python .decode()âs 'ignore' behavior
+ $checkval ||= \&_encode_check_ignore;
+
+ return ( $main, $sub, $encoding, $checkval );
+}
+
+#----------------------------------------------------------------------
+
+=head2 @lines = splitlines( $TEXT )
+
+Imitates C<str.splitlines()>. (cf. C<pydoc str>)
+
+Returns a plain list in list context. Returns the number of
+items to be returned in scalar context.
+
+=cut
+
+sub splitlines {
+ return split m<\r\n?|\n>, $_[0] if defined $_[0];
+}
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm?rev=1910959&view=auto
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm (added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Pyzor/Digest/StripHtml.pm Thu Jul 13 09:48:49 2023
@@ -0,0 +1,177 @@
+package Mail::SpamAssassin::Pyzor::Digest::StripHtml;
+
+# Copyright 2018 cPanel, LLC.
+# All rights reserved.
+# http://cpanel.net
+#
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </...@LICENSE>
+#
+
+use strict;
+use warnings;
+
+=encoding utf-8
+
+=head1 NAME
+
+Mail::SpamAssassin::Pyzor::Digest::StripHtml
+
+=head1 SYNOPSIS
+
+ my $stripped = Mail::SpamAssassin::Pyzor::Digest::StripHtml::strip($html);
+
+=head1 DESCRIPTION
+
+This module attempts to duplicate pyzorâs HTML-stripping logic.
+
+=head1 ACCURACY
+
+This library cannot achieve 100%, bug-for-bug parity with pyzor
+because to do so would require duplicating Pythonâs own HTML parsing
+library. Since that libraryâs output has changed over time, and those
+changes in turn affect pyzor, itâs literally impossible to arrive at
+a single, fully-compatible reimplementation.
+
+That said, all known divergences between pyzor and this library involve
+invalid HTML as input.
+
+Please open bug reports for any divergences you identify, particularly
+if the input is valid HTML.
+
+=cut
+
+#----------------------------------------------------------------------
+
+use HTML::Parser ();
+
+our $VERSION = '0.03';
+
+#----------------------------------------------------------------------
+
+=head1 FUNCTIONS
+
+=head2 $stripped = strip( $HTML )
+
+Give it some HTML, and itâll give back the stripped text.
+
+In B<general>, the stripping consists of removing tags as well as
+C<E<lt>scriptE<gt>> and C<E<lt>styleE<gt>> elements; however, it also
+removes HTML entities.
+
+This tries very hard to duplicate pyzorâs behavior with invalid HTML.
+
+=cut
+
+sub strip {
+ my ($html) = @_;
+
+ $html =~ s<\A\s+><>;
+ $html =~ s<\s+\z><>;
+
+ my $p = HTML::Parser->new( api_version => 3 );
+
+ my @pieces;
+
+ my $accumulate = 1;
+
+ $p->handler(
+ start => sub {
+ my ($tagname) = @_;
+
+ $accumulate = 0 if $tagname eq 'script';
+ $accumulate = 0 if $tagname eq 'style';
+
+ return;
+ },
+ 'tagname',
+ );
+
+ $p->handler(
+ end => sub {
+ $accumulate = 1;
+ return;
+ }
+ );
+
+ $p->handler(
+ text => sub {
+ my ($copy) = @_;
+
+ return if !$accumulate;
+
+ # pyzorâs HTML parser discards HTML entities. On top of that,
+ # we need to match, as closely as possible, pyzorâs handling of
+ # invalid HTML entities ⦠which is a function of Pythonâs
+ # standard HTML parsing library. This will probably never be
+ # fully compatible with the pyzor, but we can get it close.
+
+ # The original is:
+ #
+ # re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
+ #
+ # The parsing loop then âbacks upâ one byte if the last
+ # character isnât a â;â. We use a look-ahead assertion to
+ # mimic that behavior.
+ $copy =~ s<\&\# (?:[0-9]+ | [xX][0-9a-fA-F]+) (?: ; | \z | (?=[^0-9a-fA-F]) )>< >gx;
+
+ # The original is:
+ #
+ # re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
+ #
+ # We again use a look-ahead assertion to mimic Python.
+ $copy =~ s<\& [a-zA-Z] [-.a-zA-Z0-9]* (?: ; | \z | (?=[^a-zA-Z0-9]) )>< >gx;
+
+ # Pythonâs HTMLParser aborts its parsing loop when it encounters
+ # an invalid numeric reference.
+ $copy =~ s<\&\#
+ (?:
+ [^0-9xX] # anything but the expected first char
+ |
+ [0-9]+[a-fA-F] # hex within decimal
+ |
+ [xX][^0-9a-fA-F]
+ )
+ (.*)
+ ><
+ ( -1 == index($1, ';') ) ? q<> : '&#'
+ >exs;
+
+ # Pythonâs HTMLParser treats invalid entities as incomplete
+ $copy =~ s<(\&\#?)><$1 >gx;
+
+ $copy =~ s<\A\s+><>;
+ $copy =~ s<\s+\z><>;
+
+ push @pieces, \$copy if length $copy;
+ },
+ 'text,tagname',
+ );
+
+ $p->parse($html);
+ $p->eof();
+
+ my $payload = join( q< >, map { $$_ } @pieces );
+
+ # Convert all sequences of whitespace OTHER THAN non-breaking spaces to
+ # plain spaces.
+ $payload =~ s<[^\S\x{a0}]+>< >g;
+
+ return $payload;
+}
+
+1;
Modified: spamassassin/trunk/t/pyzor.t
URL: http://svn.apache.org/viewvc/spamassassin/trunk/t/pyzor.t?rev=1910959&r1=1910958&r2=1910959&view=diff
==============================================================================
--- spamassassin/trunk/t/pyzor.t (original)
+++ spamassassin/trunk/t/pyzor.t Thu Jul 13 09:48:49 2023
@@ -9,7 +9,7 @@ use constant HAS_PYZOR => Mail::SpamAss
use Test::More;
plan skip_all => "Net tests disabled" unless conf_bool('run_net_tests');
plan skip_all => "Pyzor executable not found in path" unless HAS_PYZOR;
-plan tests => 14;
+plan tests => 20;
diag('Note: Failures may not be an SpamAssassin bug, as Pyzor tests can fail due to problems with the Pyzor servers.');
@@ -43,6 +43,9 @@ ok_all_patterns();
# Same with fork
ok sarun ("--cf=\"pyzor_fork 1\" -t < data/spam/pyzor", \&patterns_run_cb);
ok_all_patterns();
+# Same with Perl
+ok sarun ("--cf=\"pyzor_fork 0\" --cf=\"pyzor_perl 1\" -t < data/spam/pyzor", \&patterns_run_cb);
+ok_all_patterns();
#TESTING FOR HAM
%patterns = (
@@ -58,4 +61,7 @@ ok_all_patterns();
# same with fork
ok sarun ("-D pyzor --cf=\"pyzor_fork 1\" -t < data/nice/001 2>&1", \&patterns_run_cb);
ok_all_patterns();
+# same with Perl
+ok sarun ("-D pyzor --cf=\"pyzor_fork 0\" --cf=\"pyzor_perl 1\" -t < data/nice/001 2>&1", \&patterns_run_cb);
+ok_all_patterns();