You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jm...@apache.org on 2004/05/15 03:33:38 UTC

svn commit: rev 10665 - in incubator/spamassassin/trunk: . lib/Mail/SpamAssassin masses

Author: jm
Date: Fri May 14 18:33:37 2004
New Revision: 10665

Modified:
   incubator/spamassassin/trunk/CREDITS
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
   incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
   incubator/spamassassin/trunk/masses/mass-check
   incubator/spamassassin/trunk/sa-learn.raw
   incubator/spamassassin/trunk/spamassassin.raw
Log:
bug 2372: added support for the parsing of mbx mailboxes (the mailbox format used within the University of Washington's IMAP implementation -- http://www.washington.edu/imap/).  Thanks to John Newman for this patch

Modified: incubator/spamassassin/trunk/CREDITS
==============================================================================
--- incubator/spamassassin/trunk/CREDITS	(original)
+++ incubator/spamassassin/trunk/CREDITS	Fri May 14 18:33:37 2004
@@ -219,6 +219,8 @@
   - Nathan Neulinger, <nneul(at)umr.edu>: security patch; code
     cleanup; dccifd support.
 
+  - John Newman, <jnewman(at)scms.waikato.ac.nz>: UW .mbx file-format support.
+
   - Mike Nolan, <nolan(at)naic.edu>: SunOS build directions
 
   - Martin O"stlund, <mo(at)microsaft.nu>: Slackware 9.0 rc-script for spamd.

Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/ArchiveIterator.pm	Fri May 14 18:33:37 2004
@@ -24,6 +24,7 @@
 use IO::Select;
 use IO::Socket;
 use Mail::SpamAssassin::Util;
+use Mail::SpamAssassin::Constants qw(:sa);
 
 use constant BIG_BYTES => 256*1024;	# 256k is a big email
 use constant BIG_LINES => BIG_BYTES/65;	# 65 bytes/line is a good approximation
@@ -265,6 +266,9 @@
 	}
 	elsif ($format eq "mbox") {
 	  $method = \&scan_mailbox;
+        }
+	elsif ($format eq "mbx") {
+	  $method = \&scan_mbx;
 	}
       }
 
@@ -400,7 +404,7 @@
     $expr = "$file";
   }
   if (!open (INPUT, $expr)) {
-    warn "unable to open $file: $!\n";
+    warn "Unable to open $file: $!\n";
     return 0;
   }
   return 1;
@@ -562,6 +566,70 @@
   }
 }
 
+sub scan_mbx {
+    my ($self, $class, $folder) = @_ ;
+    my (@files, $fp) ;
+    
+    if ($folder ne '-' && -d $folder) {
+	# got passed a directory full of mbx folders.
+	$folder =~ s/\/\s*$//; # remove trailing slash, if there is one
+	opendir(DIR, $folder) || die "Can't open '$folder' dir: $!" ;
+	while($_ = readdir(DIR)) {
+	    if(/^[^\.]\S*$/ && ! -d "$folder/$_") {
+		push(@files, "$folder/$_");
+	    }
+	}
+	closedir(DIR);
+    } else {
+	push(@files, $folder) ;
+    }
+    
+    foreach my $file (@files) {
+	if ($folder =~ /\.(?:gz|bz2)$/) {
+	    die "compressed mbx folders are not supported at this time\n" ;
+	}
+	mail_open($file) or return ;
+
+	# check the mailbox is in mbx format
+	$fp = <INPUT> ;
+	if ($fp !~ /\*mbx\*/) {
+	    die "Error, mailbox not in mbx format!\n" ;
+	}
+	
+	# skip mbx headers to the first email...
+	seek(INPUT, 2048, 0) ;
+
+        my $sep = MBX_SEPERATOR;
+    
+	while (<INPUT>) {
+	    if ($_ =~ /$sep/) {
+		my $offset = tell INPUT ;
+		my $size = $2 ;
+
+		# gather up the headers...
+		my $header = '' ;
+		while (<INPUT>) {
+		    last if (/^$/) ;
+		    $header .= $_ ;
+		}
+
+		my $t;
+		if ($self->{opt_n}) {
+		    $t = $no++;
+		} else {
+		    $t = Mail::SpamAssassin::Util::receive_date($header);
+		    next if !$self->message_is_useful_by_date($t);
+		}
+		$self->{$class}->{index_pack($class, "b", $t, "$file.$offset")} = $t;
+		seek(INPUT, $offset + $size, 0) ;
+	    } else {
+		die "Error, failure to read message body!\n" ;
+	    }
+	}
+	close INPUT;
+    }
+}
+
 ############################################################################
 
 sub run_message {
@@ -575,6 +643,9 @@
   elsif ($format eq "m") {
     return $self->run_mailbox($class, $mail, $date);
   }
+  elsif ($format eq "b") {
+    return $self->run_mbx($class, $mail, $date);
+  }
 }
 
 sub run_file {
@@ -615,7 +686,30 @@
     push (@msg, $_);
   }
   close INPUT;
-  &{$self->{wanted_sub}}($class, "$file.$offset", $date, \@msg);
+  &{$self->{wanted_sub}}($class, $where, $date, \@msg);
+}
+
+sub run_mbx {
+    my ($self, $class, $where, $date) = @_ ;
+
+    my ($file, $offset) = ($where =~ m/(.*)\.(\d+)$/) ;
+    my @msg ;
+
+    mail_open($file) or return ;
+    seek(INPUT, $offset, 0) ;
+    
+    while (<INPUT>) {
+	last if ($_ =~ MBX_SEPERATOR) ;
+	
+	# skip mails that are too big
+	if (! $self->{opt_all} && @msg > BIG_LINES) {
+	    close INPUT ;
+	    return ;
+	}
+	push (@msg, $_) ;
+    }
+    close INPUT ;
+    &{$self->{wanted_sub}}($class, $where, $date, \@msg) ;
 }
 
 ############################################################################

Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Constants.pm	Fri May 14 18:33:37 2004
@@ -34,7 +34,7 @@
 );
 # These are generic constants that may be used across several modules
 @SA_VARS = qw(
-	META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY
+	META_TEST_MIN_PRIORITY HARVEST_DNSBL_PRIORITY MBX_SEPERATOR
 );
 
 %EXPORT_TAGS = (
@@ -167,5 +167,13 @@
 
 use constant META_TEST_MIN_PRIORITY => 500;
 use constant HARVEST_DNSBL_PRIORITY => 500;
+
+# Regular expression that matches message sperators in The University of
+# Washington's MBX mailbox format...
+use constant MBX_SEPERATOR => qr/([\s|\d]\d-[a-zA-Z]{3}-\d{4}\s\d{2}:\d{2}:\d{2}.*),(\d+);([\da-f]{12})-(\w{8})/ ;
+# $1 = datestamp (str)
+# $2 = size of message in bytes (int)
+# $3 = message status - binary (hex)
+# $4 = message ID (hex)
 
 1;

Modified: incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm	(original)
+++ incubator/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm	Fri May 14 18:33:37 2004
@@ -48,6 +48,7 @@
 use Mail::SpamAssassin;
 use Mail::SpamAssassin::Message::Node;
 use Mail::SpamAssassin::Message::Metadata;
+use Mail::SpamAssassin::Constants qw(:sa);
 
 use vars qw(@ISA);
 
@@ -78,6 +79,13 @@
 
 =cut
 
+# month mappings (ripped from Util.pm)
+my %MONTH = (jan => 1, feb => 2, mar => 3, apr => 4, may => 5, jun => 6,
+	     jul => 7, aug => 8, sep => 9, oct => 10, nov => 11, dec => 12);
+
+# day of week mapping (starting from zero)
+my @DAY_OF_WEEK = qw/Sun Mon Tue Wed Thu Fri Sat/ ;
+
 sub new {
   my $class = shift;
   $class = ref($class) || $class;
@@ -116,10 +124,38 @@
 
   # Go through all the headers of the message
   my $header = '';
+  my $mbxsep = MBX_SEPERATOR;
   while ( my $last = shift @message ) {
     if ( $last =~ /^From\s/ ) {
-      $self->{'mbox_sep'} = $last;
-      next;
+	# mbox formated mailbox
+	$self->{'mbox_sep'} = $last;
+	next;
+    } elsif ($last =~ /$mbxsep/) {
+	# Munge the mbx message seperator into mbox format as a sort of
+	# de facto portability standard in SA's internals.  We need to
+	# to this so that Mail::SpamAssassin::Util::parse_rfc822_date
+	# can parse the date string...
+	if (/([\s|\d]\d)-([a-zA-Z]{3})-(\d{4})\s(\d{2}):(\d{2}):(\d{2})/o) {
+	    # $1 = day of month
+	    # $2 = month (text)
+	    # $3 = year
+	    # $4 = hour
+	    # $5 = min
+	    # $6 = sec
+	    my @arr = localtime(timelocal($6,$5,$4,$1,$MONTH{lc($2)}-1,$3)) ;
+	    my $address ;
+	    foreach (@message) {
+		if (/From:\s[^<]+<([^>]+)>/) {
+		    $address = $1 ;
+		    last ;
+		} elsif (/From:\s([^<^>]+)/) {
+		    $address = $1 ;
+		    last ;
+		}
+	    }
+	    $self->{'mbox_sep'} = "From $address $DAY_OF_WEEK[$arr[6]] $2 $1 $4:$5:$6 $3\n" ;
+	    next;
+	}
     }
 
     # Store the non-modified headers in a scalar

Modified: incubator/spamassassin/trunk/masses/mass-check
==============================================================================
--- incubator/spamassassin/trunk/masses/mass-check	(original)
+++ incubator/spamassassin/trunk/masses/mass-check	Fri May 14 18:33:37 2004
@@ -57,6 +57,7 @@
   --dir         subsequent targets are directories
   --file        subsequent targets are files in RFC 822 format
   --mbox        subsequent targets are mbox files
+  --mbx         subsequent targets are mbx files
  
   Just left over functions we should remove at some point:
   --bayes       report score from Bayesian classifier
@@ -64,7 +65,7 @@
   non-option arguments are used as target names (mail files and folders),
   the target format is: <class>:<format>:<location>
   <class>       is "spam" or "ham"
-  <format>      is "dir", "file", or "mbox"
+  <format>      is "dir", "file", "mbx", or "mbox"
   <location>    is a file or directory name.  globbing of ~ and * is supported
 
 ENDOFUSAGE
@@ -103,6 +104,7 @@
 	   "dir" => sub { $opt_format = "dir"; },
 	   "file" => sub { $opt_format = "file"; },
 	   "mbox" => sub { $opt_format = "mbox"; },
+	   "mbx" => sub { $opt_format = "mbx"; },
 	   '<>' => \&target) or usage();
 
 if ($opt_f) {

Modified: incubator/spamassassin/trunk/sa-learn.raw
==============================================================================
--- incubator/spamassassin/trunk/sa-learn.raw	(original)
+++ incubator/spamassassin/trunk/sa-learn.raw	Fri May 14 18:33:37 2004
@@ -126,6 +126,7 @@
   'dir'    => sub { $opt{'old_format'} = 'dir'; },
   'file'   => sub { $opt{'old_format'} = 'file'; },
   'mbox'   => sub { $opt{'format'}     = 'mbox'; },
+  'mbx'    => sub { $opt{'format'}     = 'mbx'; },
   'single' => sub { $opt{'old_format'} = 'single'; },
 
   'db|dbpath=s' => \$bayes_override_path,
@@ -171,7 +172,7 @@
 
 if ( defined $opt{'old_format'} ) {
 
-  #Format specified in the 2.5x form of --dir, --file, --mbox or --single.
+  #Format specified in the 2.5x form of --dir, --file, --mbox, --mbx or --single.
   #Convert it to the new behavior:
   if ( $opt{'old_format'} eq 'single' ) {
     push ( @ARGV, '-' );
@@ -350,11 +351,11 @@
     target('-');
   }
 
-  # mbox doesn't deal with STDIN, so make a temp file if they want STDIN.
+  # mbox and mbx doesn't deal with STDIN, so make a temp file if they want STDIN.
   # do it here since they may specify "-" on the commandline
   #
   my $tempfile;
-  if ( $targets[0] =~ /:mbox:-$/ ) {
+  if ( $targets[0] =~ /:mbo?x:-$/ ) {
     my $handle;
 
     local $/ = undef;    # go into slurp mode
@@ -503,6 +504,7 @@
  --dir                             Ignored; historical compatability
  --file                            Ignored; historical compatability
  --mbox                            Input sources are in mbox format
+ --mbx                             Input sources are in mbx format
  --showdots                        Show progress using dots
  --no-sync                         Skip syncronizing the journal and
                                    databases after scan

Modified: incubator/spamassassin/trunk/spamassassin.raw
==============================================================================
--- incubator/spamassassin/trunk/spamassassin.raw	(original)
+++ incubator/spamassassin/trunk/spamassassin.raw	Fri May 14 18:33:37 2004
@@ -125,6 +125,7 @@
   'version|V'                               => \$opt{'version'},
   'help|h|?'                                => \$opt{'help'},
   'mbox'                                    => sub { $opt{'format'} = 'mbox'; },
+  'mbx'                                     => sub { $opt{'format'} = 'mbx'; },
 
   # NOTE: ignored for backwards compatibility. will be stripped
   # in a future release.
@@ -231,12 +232,12 @@
 push ( @ARGV, '-' ) if ( !@ARGV );
 @targets = map { join ( ":", '', $opt{'format'}, $_ ) } @ARGV;
 
-# mbox handling doesn't deal with STDIN, so make a temp file if they
+# mbox and mbx handling doesn't deal with STDIN, so make a temp file if they
 # want STDIN.  do it here since they may specify "-" on the commandline
 # instead of leaving it as the default.
 #
 my $tempfile;
-if ( $targets[0] =~ /:mbox:-$/ ) {
+if ( $targets[0] =~ /:mbo?x:-$/ ) {
   my $handle;
 
   local $/ = undef;    # go into slurp mode
@@ -407,6 +408,8 @@
                                    tested message was spam
  --mbox [filename]                 read in messages from a mbox file, given
                                    by 'filename' or STDIN otherwise.
+ --mbx [filename]                  read in messages from a UW mbx file, given
+                                   by 'filename' or STDIN otherwise.
  -t, --test-mode                   Pipe message through and add extra
                                    report to the bottom
  --lint                            Lint the rule set: report syntax errors
@@ -593,6 +596,13 @@
 
 Specify that the input message(s) are in mbox format.  If I<filename> is
 given, messages will be read in from that file.  Otherwise, STDIN is used.
+
+=item B<--mbx> [filename]
+
+Specify that the input message(s) are in UW .mbx format.  If I<filename> is
+given, messages will be read in from that file.  Otherwise, STDIN is used. mbx
+is the mailbox format used within the University of Washington's IMAP
+implementation; see http://www.washington.edu/imap/ .
 
 =back