You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jg...@apache.org on 2006/01/07 00:09:12 UTC

svn commit: r366594 - in /spamassassin/trunk: INSTALL lib/Mail/SpamAssassin/Message.pm lib/Mail/SpamAssassin/Message/Node.pm lib/Mail/SpamAssassin/Util/DependencyInfo.pm t/memory_cycles.t

Author: jgmyers
Date: Fri Jan  6 15:09:10 2006
New Revision: 366594

URL: http://svn.apache.org/viewcvs?rev=366594&view=rev
Log:
Bug 4636: Add support for charset normalization

Modified:
    spamassassin/trunk/INSTALL
    spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
    spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
    spamassassin/trunk/t/memory_cycles.t

Modified: spamassassin/trunk/INSTALL
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/INSTALL?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/INSTALL (original)
+++ spamassassin/trunk/INSTALL Fri Jan  6 15:09:10 2006
@@ -327,6 +327,10 @@
 
     Used by sa-update to expand update archives.
 
+  - Encode::Detect (from CPAN)
+
+    If this module is installed, SpamAssassin will detect charsets
+    and convert them into Unicode.
 
 What Next?
 ----------

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Fri Jan  6 15:09:10 2006
@@ -772,6 +772,7 @@
   # 0: content-type, 1: boundary, 2: charset, 3: filename
   my @ct = Mail::SpamAssassin::Util::parse_content_type($part_msg->header('content-type'));
   $part_msg->{'type'} = $ct[0];
+  $part_msg->{'charset'} = $ct[2];
 
   # multipart sections are required to have a boundary set ...  If this
   # one doesn't, assume it's malformed and revert to text/plain

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm Fri Jan  6 15:09:10 2006
@@ -41,6 +41,8 @@
 use Mail::SpamAssassin::HTML;
 use Mail::SpamAssassin::Logger;
 
+our $normalize_supported = ( $] > 5.008004 && eval 'require Encode::Detect::Detector' && eval 'require Encode' );
+
 =item new()
 
 Generates an empty Node object and returns it.  Typically only called
@@ -341,6 +343,30 @@
   return 0;
 }
 
+sub _normalize {
+  my ($data, $charset) = @_;
+  return $data unless $normalize_supported;
+  my $detected = Encode::Detect::Detector::detect($data);
+
+  my $converter;
+
+  if ($charset && $charset !~ /^us-ascii$/i &&
+      ($detected || 'none') !~ /^(?:UTF|EUC|ISO-2022|Shift_JIS|Big5|GB)/i) {
+      dbg("Using labeled charset $charset");
+      $converter = Encode::find_encoding($charset);
+  }
+
+  $converter = Encode::find_encoding($detected) unless $converter || !defined($detected);
+
+  return $data unless $converter;
+
+  dbg("Converting...");
+
+  my $rv = $converter->decode($data, 0);
+  utf8::downgrade($rv, 1);
+  return $rv
+}
+
 =item rendered()
 
 render_text() takes the given text/* type MIME part, and attempts to
@@ -358,7 +384,7 @@
   return(undef,undef) unless ( $self->{'type'} =~ /^text\b/i );
 
   if (!exists $self->{rendered}) {
-    my $text = $self->decode();
+    my $text = _normalize($self->decode(), $self->{charset});
     my $raw = length($text);
 
     # render text/html always, or any other text|text/plain part as text/html
@@ -385,7 +411,7 @@
     }
     else {
       $self->{rendered_type} = $self->{type};
-      $self->{rendered} = $text;
+      $self->{rendered} = $self->{visible_rendered} = $text;
     }
   }
 
@@ -477,7 +503,7 @@
 
   if ( $cte eq 'B' ) {
     # base 64 encoded
-    return Mail::SpamAssassin::Util::base64_decode($data);
+    $data = Mail::SpamAssassin::Util::base64_decode($data);
   }
   elsif ( $cte eq 'Q' ) {
     # quoted printable
@@ -485,12 +511,13 @@
     # the RFC states that in the encoded text, "_" is equal to "=20"
     $data =~ s/_/=20/g;
 
-    return Mail::SpamAssassin::Util::qp_decode($data);
+    $data = Mail::SpamAssassin::Util::qp_decode($data);
   }
   else {
     # not possible since the input has already been limited to 'B' and 'Q'
     die "message: unknown encoding type '$cte' in RFC2047 header";
   }
+  return _normalize($data, $encoding);
 }
 
 # Decode base64 and quoted-printable in headers according to RFC2047.
@@ -504,15 +531,15 @@
   $header =~ s/\n[ \t]+/\n /g;
   $header =~ s/\r?\n//g;
 
-  return $header unless $header =~ /=\?/;
-
   # multiple encoded sections must ignore the interim whitespace.
   # to avoid possible FPs with (\s+(?==\?))?, look for the whole RE
   # separated by whitespace.
   1 while ($header =~ s/(=\?[\w_-]+\?[bqBQ]\?[^?]+\?=)\s+(=\?[\w_-]+\?[bqBQ]\?[^?]+\?=)/$1$2/g);
 
-  $header =~
-    s/=\?([\w_-]+)\?([bqBQ])\?([^?]+)\?=/__decode_header($1, uc($2), $3)/ge;
+  unless ($header =~
+	  s/=\?([\w_-]+)\?([bqBQ])\?([^?]+)\?=/__decode_header($1, uc($2), $3)/ge) {
+    $header = _normalize($header);
+  }
 
   return $header;
 }

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm Fri Jan  6 15:09:10 2006
@@ -166,6 +166,12 @@
   desc => 'The "sa-update" script requires this module to access compressed
   update archive files.',
 },
+{
+  module => 'Encode::Detect',
+  version => '0.00',
+  desc => 'If this module is installed, SpamAssassin will detect charsets
+  and convert them into Unicode.',
+},
 );
 
 ###########################################################################

Modified: spamassassin/trunk/t/memory_cycles.t
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/t/memory_cycles.t?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/t/memory_cycles.t (original)
+++ spamassassin/trunk/t/memory_cycles.t Fri Jan  6 15:09:10 2006
@@ -21,7 +21,7 @@
 use SATest; sa_t_init("memory_cycles");
 
 use Test; BEGIN {
-  plan tests => (HAVE_DEVEL_CYCLE ? 3 : 0);
+  plan tests => (HAVE_DEVEL_CYCLE ? 4 : 0);
 }
 unless (HAVE_DEVEL_CYCLE) {
   print "# Devel::Cycle module required for this test, skipped\n";
@@ -55,8 +55,12 @@
 $status->finish();
 ok (check_for_cycles($status));
 
+$mail->finish();
+ok (check_for_cycles($mail));
+
 $spamtest->finish();
 ok (check_for_cycles($spamtest));
+
 exit;
 
 ############################################################################