You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2004/05/15 03:33:38 UTC
svn commit: rev 10665 - in incubator/spamassassin/trunk: . lib/Mail/SpamAssassin masses
Author: jm
Date: Fri May 14 18:33:37 2004
New Revision: 10665
Modified:
incubator/spamassassin/trunk/CREDITS
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
incubator/spamassassin/trunk/masses/mass-check
incubator/spamassassin/trunk/sa-learn.raw
incubator/spamassassin/trunk/spamassassin.raw
Log:
bug 2372: added support for the parsing of mbx mailboxes (the mailbox format used within the University of Washington's IMAP implementation -- http://www.washington.edu/imap/). Thanks to John Newman for this patch
Modified: incubator/spamassassin/trunk/CREDITS
==============================================================================
--- incubator/spamassassin/trunk/CREDITS (original)
+++ incubator/spamassassin/trunk/CREDITS Fri May 14 18:33:37 2004
@@ -219,6 +219,8 @@
- Nathan Neulinger, <nneul(at)umr.edu>: security patch; code
cleanup; dccifd support.
+ - John Newman, <jnewman(at)scms.waikato.ac.nz>: UW .mbx file-format support.
+
- Mike Nolan, <nolan(at)naic.edu>: SunOS build directions
- Martin O"stlund, <mo(at)microsaft.nu>: Slackware 9.0 rc-script for spamd.
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm Fri May 14 18:33:37 2004
@@ -24,6 +24,7 @@
use IO::Select;
use IO::Socket;
use Mail::SpamAssassin::Util;
+use Mail::SpamAssassin::Constants qw(:sa);
use constant BIG_BYTES => 256*1024; # 256k is a big email
use constant BIG_LINES => BIG_BYTES/65; # 65 bytes/line is a good approximation
@@ -265,6 +266,9 @@
}
elsif ($format eq "mbox") {
$method = \&scan_mailbox;
+ }
+ elsif ($format eq "mbx") {
+ $method = \&scan_mbx;
}
}
@@ -400,7 +404,7 @@
$expr = "$file";
}
if (!open (INPUT, $expr)) {
- warn "unable to open $file: $!\n";
+ warn "Unable to open $file: $!\n";
return 0;
}
return 1;
@@ -562,6 +566,70 @@
}
}
+sub scan_mbx {
+ my ($self, $class, $folder) = @_ ;
+ my (@files, $fp) ;
+
+ if ($folder ne '-' && -d $folder) {
+ # got passed a directory full of mbx folders.
+ $folder =~ s/\/\s*$//; # remove trailing slash, if there is one
+ opendir(DIR, $folder) || die "Can't open '$folder' dir: $!" ;
+ while($_ = readdir(DIR)) {
+ if(/^[^\.]\S*$/ && ! -d "$folder/$_") {
+ push(@files, "$folder/$_");
+ }
+ }
+ closedir(DIR);
+ } else {
+ push(@files, $folder) ;
+ }
+
+ foreach my $file (@files) {
+ if ($folder =~ /\.(?:gz|bz2)$/) {
+ die "compressed mbx folders are not supported at this time\n" ;
+ }
+ mail_open($file) or return ;
+
+ # check the mailbox is in mbx format
+ $fp = <INPUT> ;
+ if ($fp !~ /\*mbx\*/) {
+ die "Error, mailbox not in mbx format!\n" ;
+ }
+
+ # skip mbx headers to the first email...
+ seek(INPUT, 2048, 0) ;
+
+ my $sep = MBX_SEPERATOR;
+
+ while (<INPUT>) {
+ if ($_ =~ /$sep/) {
+ my $offset = tell INPUT ;
+ my $size = $2 ;
+
+ # gather up the headers...
+ my $header = '' ;
+ while (<INPUT>) {
+ last if (/^$/) ;
+ $header .= $_ ;
+ }
+
+ my $t;
+ if ($self->{opt_n}) {
+ $t = $no++;
+ } else {
+ $t = Mail::SpamAssassin::Util::receive_date($header);
+ next if !$self->message_is_useful_by_date($t);
+ }
+ $self->{$class}->{index_pack($class, "b", $t, "$file.$offset")} = $t;
+ seek(INPUT, $offset + $size, 0) ;
+ } else {
+ die "Error, failure to read message body!\n" ;
+ }
+ }
+ close INPUT;
+ }
+}
+
############################################################################
sub run_message {
@@ -575,6 +643,9 @@
elsif ($format eq "m") {
return $self->run_mailbox($class, $mail, $date);
}
+ elsif ($format eq "b") {
+ return $self->run_mbx($class, $mail, $date);
+ }
}
sub run_file {
@@ -615,7 +686,30 @@
push (@msg, $_);
}
close INPUT;
- &{$self->{wanted_sub}}($class, "$file.$offset", $date, \@msg);
+ &{$self->{wanted_sub}}($class, $where, $date, \@msg);
+}
+
+sub run_mbx {
+ my ($self, $class, $where, $date) = @_ ;
+
+ my ($file, $offset) = ($where =~ m/(.*)\.(\d+)$/) ;
+ my @msg ;
+
+ mail_open($file) or return ;
+ seek(INPUT, $offset, 0) ;
+
+ while (<INPUT>) {
+ last if ($_ =~ MBX_SEPERATOR) ;
+
+ # skip mails that are too big
+ if (! $self->{opt_all} && @msg > BIG_LINES) {
+ close INPUT ;
+ return ;
+ }
+ push (@msg, $_) ;
+ }
+ close INPUT ;
+ &{$self->{wanted_sub}}($class, $where, $date, \@msg) ;
}
############################################################################
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm Fri May 14 18:33:37 2004
@@ -34,7 +34,7 @@
);
# These are generic constants that may be used across several modules
@SA_VARS = qw(
- META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY
+ META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY MBX_SEPERATOR
);
%EXPORT_TAGS = (
@@ -167,5 +167,13 @@
use constant META_TEST_MIN_PRIORITY => 500;
use constant HARVEST_DNSBL_PRIORITY => 500;
+
+# Regular expression that matches message sperators in The University of
+# Washington's MBX mailbox format...
+use constant MBX_SEPERATOR => qr/([\s|\d]\d-[a-zA-Z]{3}-\d{4}\s\d{2}:\d{2}:\d{2}.*),(\d+);([\da-f]{12})-(\w{8})/ ;
+# $1 = datestamp (str)
+# $2 = size of message in bytes (int)
+# $3 = message status - binary (hex)
+# $4 = message ID (hex)
1;
Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm (original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Fri May 14 18:33:37 2004
@@ -48,6 +48,7 @@
use Mail::SpamAssassin;
use Mail::SpamAssassin::Message::Node;
use Mail::SpamAssassin::Message::Metadata;
+use Mail::SpamAssassin::Constants qw(:sa);
use vars qw(@ISA);
@@ -78,6 +79,13 @@
=cut
+# month mappings (ripped from Util.pm)
+my %MONTH = (jan => 1, feb => 2, mar => 3, apr => 4, may => 5, jun => 6,
+ jul => 7, aug => 8, sep => 9, oct => 10, nov => 11, dec => 12);
+
+# day of week mapping (starting from zero)
+my @DAY_OF_WEEK = qw/Sun Mon Tue Wed Thu Fri Sat/ ;
+
sub new {
my $class = shift;
$class = ref($class) || $class;
@@ -116,10 +124,38 @@
# Go through all the headers of the message
my $header = '';
+ my $mbxsep = MBX_SEPERATOR;
while ( my $last = shift @message ) {
if ( $last =~ /^From\s/ ) {
- $self->{'mbox_sep'} = $last;
- next;
+ # mbox formated mailbox
+ $self->{'mbox_sep'} = $last;
+ next;
+ } elsif ($last =~ /$mbxsep/) {
+ # Munge the mbx message seperator into mbox format as a sort of
+ # de facto portability standard in SA's internals. We need to
+ # to this so that Mail::SpamAssassin::Util::parse_rfc822_date
+ # can parse the date string...
+ if (/([\s|\d]\d)-([a-zA-Z]{3})-(\d{4})\s(\d{2}):(\d{2}):(\d{2})/o) {
+ # $1 = day of month
+ # $2 = month (text)
+ # $3 = year
+ # $4 = hour
+ # $5 = min
+ # $6 = sec
+ my @arr = localtime(timelocal($6,$5,$4,$1,$MONTH{lc($2)}-1,$3)) ;
+ my $address ;
+ foreach (@message) {
+ if (/From:\s[^<]+<([^>]+)>/) {
+ $address = $1 ;
+ last ;
+ } elsif (/From:\s([^<^>]+)/) {
+ $address = $1 ;
+ last ;
+ }
+ }
+ $self->{'mbox_sep'} = "From $address $DAY_OF_WEEK[$arr[6]] $2 $1 $4:$5:$6 $3\n" ;
+ next;
+ }
}
# Store the non-modified headers in a scalar
Modified: incubator/spamassassin/trunk/masses/mass-check
==============================================================================
--- incubator/spamassassin/trunk/masses/mass-check (original)
+++ incubator/spamassassin/trunk/masses/mass-check Fri May 14 18:33:37 2004
@@ -57,6 +57,7 @@
--dir subsequent targets are directories
--file subsequent targets are files in RFC 822 format
--mbox subsequent targets are mbox files
+ --mbx subsequent targets are mbx files
Just left over functions we should remove at some point:
--bayes report score from Bayesian classifier
@@ -64,7 +65,7 @@
non-option arguments are used as target names (mail files and folders),
the target format is: <class>:<format>:<location>
<class> is "spam" or "ham"
- <format> is "dir", "file", or "mbox"
+ <format> is "dir", "file", "mbx", or "mbox"
<location> is a file or directory name. globbing of ~ and * is supported
ENDOFUSAGE
@@ -103,6 +104,7 @@
"dir" => sub { $opt_format = "dir"; },
"file" => sub { $opt_format = "file"; },
"mbox" => sub { $opt_format = "mbox"; },
+ "mbx" => sub { $opt_format = "mbx"; },
'<>' => \&target) or usage();
if ($opt_f) {
Modified: incubator/spamassassin/trunk/sa-learn.raw
==============================================================================
--- incubator/spamassassin/trunk/sa-learn.raw (original)
+++ incubator/spamassassin/trunk/sa-learn.raw Fri May 14 18:33:37 2004
@@ -126,6 +126,7 @@
'dir' => sub { $opt{'old_format'} = 'dir'; },
'file' => sub { $opt{'old_format'} = 'file'; },
'mbox' => sub { $opt{'format'} = 'mbox'; },
+ 'mbx' => sub { $opt{'format'} = 'mbx'; },
'single' => sub { $opt{'old_format'} = 'single'; },
'db|dbpath=s' => \$bayes_override_path,
@@ -171,7 +172,7 @@
if ( defined $opt{'old_format'} ) {
- #Format specified in the 2.5x form of --dir, --file, --mbox or --single.
+ #Format specified in the 2.5x form of --dir, --file, --mbox, --mbx or --single.
#Convert it to the new behavior:
if ( $opt{'old_format'} eq 'single' ) {
push ( @ARGV, '-' );
@@ -350,11 +351,11 @@
target('-');
}
- # mbox doesn't deal with STDIN, so make a temp file if they want STDIN.
+ # mbox and mbx doesn't deal with STDIN, so make a temp file if they want STDIN.
# do it here since they may specify "-" on the commandline
#
my $tempfile;
- if ( $targets[0] =~ /:mbox:-$/ ) {
+ if ( $targets[0] =~ /:mbo?x:-$/ ) {
my $handle;
local $/ = undef; # go into slurp mode
@@ -503,6 +504,7 @@
--dir Ignored; historical compatability
--file Ignored; historical compatability
--mbox Input sources are in mbox format
+ --mbx Input sources are in mbx format
--showdots Show progress using dots
--no-sync Skip syncronizing the journal and
databases after scan
Modified: incubator/spamassassin/trunk/spamassassin.raw
==============================================================================
--- incubator/spamassassin/trunk/spamassassin.raw (original)
+++ incubator/spamassassin/trunk/spamassassin.raw Fri May 14 18:33:37 2004
@@ -125,6 +125,7 @@
'version|V' => \$opt{'version'},
'help|h|?' => \$opt{'help'},
'mbox' => sub { $opt{'format'} = 'mbox'; },
+ 'mbx' => sub { $opt{'format'} = 'mbx'; },
# NOTE: ignored for backwards compatibility. will be stripped
# in a future release.
@@ -231,12 +232,12 @@
push ( @ARGV, '-' ) if ( !@ARGV );
@targets = map { join ( ":", '', $opt{'format'}, $_ ) } @ARGV;
-# mbox handling doesn't deal with STDIN, so make a temp file if they
+# mbox and mbx handling doesn't deal with STDIN, so make a temp file if they
# want STDIN. do it here since they may specify "-" on the commandline
# instead of leaving it as the default.
#
my $tempfile;
-if ( $targets[0] =~ /:mbox:-$/ ) {
+if ( $targets[0] =~ /:mbo?x:-$/ ) {
my $handle;
local $/ = undef; # go into slurp mode
@@ -407,6 +408,8 @@
tested message was spam
--mbox [filename] read in messages from a mbox file, given
by 'filename' or STDIN otherwise.
+ --mbx [filename] read in messages from a UW mbx file, given
+ by 'filename' or STDIN otherwise.
-t, --test-mode Pipe message through and add extra
report to the bottom
--lint Lint the rule set: report syntax errors
@@ -593,6 +596,13 @@
Specify that the input message(s) are in mbox format. If I<filename> is
given, messages will be read in from that file. Otherwise, STDIN is used.
+
+=item B<--mbx> [filename]
+
+Specify that the input message(s) are in UW .mbx format. If I<filename> is
+given, messages will be read in from that file. Otherwise, STDIN is used. mbx
+is the mailbox format used within the University of Washington's IMAP
+implementation; see http://www.washington.edu/imap/ .
=back