svn commit: r1613554 - /spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm

[km...@apache.org]

Author: kmcgrail
Date: Sat Jul 26 02:24:44 2014
New Revision: 1613554

URL: http://svn.apache.org/r1613554
Log:
Bug 7068 - rule and code to counts unicode entities

Modified:
    spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm?rev=1613554&r1=1613553&r2=1613554&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/MIMEEval.pm Sat Jul 26 02:24:44 2014
@@ -48,6 +48,7 @@ sub new {
   $self->register_eval_rule("check_mime_multipart_ratio");
   $self->register_eval_rule("check_msg_parse_flags");
   $self->register_eval_rule("check_for_ascii_text_illegal");
+  $self->register_eval_rule("check_abundant_unicode_ratio");
   $self->register_eval_rule("check_for_faraway_charset");
   $self->register_eval_rule("check_for_uppercase");
   $self->register_eval_rule("check_ma_non_text");
@@ -78,6 +79,32 @@ sub check_for_ascii_text_illegal {
   return ($pms->{mime_ascii_text_illegal} > 0);
 }
 
+=item has_check_abundant_unicode_ratio
+
+Adds capability check for "if can()" for check_abundant_unicode_ratio
+
+=cut
+
+sub has_check_abundant_unicode_ratio { 1 }
+
+=item check_abundant_unicode_ratio
+
+A MIME part claiming to be text/plain and containing Unicode characters must be encoded as quoted-printable or base64.  Any message in 7bit or 8bit encoding containing (HTML) Unicode entities will not render them as Unicode, but literally.
+
+Thus a few such sequences might occur on a mailing list of developers discussing such characters, but a message with a high density of such characters is likely spam.
+
+=cut
+
+sub check_abundant_unicode_ratio {
+  my ($self, $pms, undef, $ratio) = @_;
+
+  # validate ratio?
+  return 0 unless ($ratio =~ /^\d{0,3}\.\d{1,3}$/);
+
+  $self->_check_attachments($pms) unless exists $pms->{mime_text_unicode_ratio};
+  return ($pms->{mime_text_unicode_ratio} >= $ratio);
+}
+
 sub check_for_faraway_charset {
   my ($self, $pms, $body) = @_;
 
@@ -232,6 +259,9 @@ sub _check_attachments {
   my @part_bytes;		# MIME part total bytes
   my @part_type;		# MIME part types
 
+  my $normal_chars = 0;		# MIME text bytes that aren't encoded
+  my $unicode_chars = 0;	# MIME text bytes that are unicode entities
+
   # MIME header information
   my $part = -1;		# MIME part index
 
@@ -257,6 +287,7 @@ sub _check_attachments {
   $pms->{mime_qp_long_line} = 0;
   $pms->{mime_qp_ratio} = 0;
   $pms->{mime_ascii_text_illegal} = 0;
+  $pms->{mime_text_unicode_ratio} = 0;
 
   # Get all parts ...
   foreach my $p ($pms->{msg}->find_parts(qr/./)) {
@@ -364,6 +395,23 @@ sub _check_attachments {
         }
       }
 
+      # if we're text/plain, we should never see unicode escapes in this
+      # format, especially not for 7bit or 8bit. unicode requires base64
+      # or quoted-printable encoding.
+      if ($ctype eq 'text/plain' && ($cte eq '' || $cte eq '7bit' || $cte eq '8bit')) {
+        my ($text, $subs) = $_;
+
+        $subs = $text =~ s/&#x[0-9A-F]{4};//g;
+        $normal_chars += length($text);
+        $unicode_chars += $subs;
+
+        if ($subs && would_log('dbg', 'eval')) {
+          my $str = $_;
+          $str = substr($str, 0, 512) . '...' if (length($str) > 512);
+          dbg("check: abundant_unicode: " . $str . " (" . $subs . ")\n");
+        }
+      }
+
       $previous = $_;
     }
   }
@@ -374,6 +422,10 @@ sub _check_attachments {
     $pms->{mime_qp_bytes} = $qp_bytes;
   }
 
+  if ($normal_chars) {
+    $pms->{mime_text_unicode_ratio} = $unicode_chars / $normal_chars;
+  }
+
   if ($pms->{mime_multipart_alternative}) {
     my $text;
     my $html;