You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by jg...@apache.org on 2006/01/07 00:09:12 UTC
svn commit: r366594 - in /spamassassin/trunk: INSTALL
lib/Mail/SpamAssassin/Message.pm lib/Mail/SpamAssassin/Message/Node.pm
lib/Mail/SpamAssassin/Util/DependencyInfo.pm t/memory_cycles.t
Author: jgmyers
Date: Fri Jan 6 15:09:10 2006
New Revision: 366594
URL: http://svn.apache.org/viewcvs?rev=366594&view=rev
Log:
Bug 4636: Add support for charset normalization
Modified:
spamassassin/trunk/INSTALL
spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
spamassassin/trunk/t/memory_cycles.t
Modified: spamassassin/trunk/INSTALL
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/INSTALL?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/INSTALL (original)
+++ spamassassin/trunk/INSTALL Fri Jan 6 15:09:10 2006
@@ -327,6 +327,10 @@
Used by sa-update to expand update archives.
+ - Encode::Detect (from CPAN)
+
+ If this module is installed, SpamAssassin will detect charsets
+ and convert them into Unicode.
What Next?
----------
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Fri Jan 6 15:09:10 2006
@@ -772,6 +772,7 @@
# 0: content-type, 1: boundary, 2: charset, 3: filename
my @ct = Mail::SpamAssassin::Util::parse_content_type($part_msg->header('content-type'));
$part_msg->{'type'} = $ct[0];
+ $part_msg->{'charset'} = $ct[2];
# multipart sections are required to have a boundary set ... If this
# one doesn't, assume it's malformed and revert to text/plain
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm Fri Jan 6 15:09:10 2006
@@ -41,6 +41,8 @@
use Mail::SpamAssassin::HTML;
use Mail::SpamAssassin::Logger;
+our $normalize_supported = ( $] > 5.008004 && eval 'require Encode::Detect::Detector' && eval 'require Encode' );
+
=item new()
Generates an empty Node object and returns it. Typically only called
@@ -341,6 +343,30 @@
return 0;
}
+sub _normalize {
+ my ($data, $charset) = @_;
+ return $data unless $normalize_supported;
+ my $detected = Encode::Detect::Detector::detect($data);
+
+ my $converter;
+
+ if ($charset && $charset !~ /^us-ascii$/i &&
+ ($detected || 'none') !~ /^(?:UTF|EUC|ISO-2022|Shift_JIS|Big5|GB)/i) {
+ dbg("Using labeled charset $charset");
+ $converter = Encode::find_encoding($charset);
+ }
+
+ $converter = Encode::find_encoding($detected) unless $converter || !defined($detected);
+
+ return $data unless $converter;
+
+ dbg("Converting...");
+
+ my $rv = $converter->decode($data, 0);
+ utf8::downgrade($rv, 1);
+ return $rv
+}
+
=item rendered()
render_text() takes the given text/* type MIME part, and attempts to
@@ -358,7 +384,7 @@
return(undef,undef) unless ( $self->{'type'} =~ /^text\b/i );
if (!exists $self->{rendered}) {
- my $text = $self->decode();
+ my $text = _normalize($self->decode(), $self->{charset});
my $raw = length($text);
# render text/html always, or any other text|text/plain part as text/html
@@ -385,7 +411,7 @@
}
else {
$self->{rendered_type} = $self->{type};
- $self->{rendered} = $text;
+ $self->{rendered} = $self->{visible_rendered} = $text;
}
}
@@ -477,7 +503,7 @@
if ( $cte eq 'B' ) {
# base 64 encoded
- return Mail::SpamAssassin::Util::base64_decode($data);
+ $data = Mail::SpamAssassin::Util::base64_decode($data);
}
elsif ( $cte eq 'Q' ) {
# quoted printable
@@ -485,12 +511,13 @@
# the RFC states that in the encoded text, "_" is equal to "=20"
$data =~ s/_/=20/g;
- return Mail::SpamAssassin::Util::qp_decode($data);
+ $data = Mail::SpamAssassin::Util::qp_decode($data);
}
else {
# not possible since the input has already been limited to 'B' and 'Q'
die "message: unknown encoding type '$cte' in RFC2047 header";
}
+ return _normalize($data, $encoding);
}
# Decode base64 and quoted-printable in headers according to RFC2047.
@@ -504,15 +531,15 @@
$header =~ s/\n[ \t]+/\n /g;
$header =~ s/\r?\n//g;
- return $header unless $header =~ /=\?/;
-
# multiple encoded sections must ignore the interim whitespace.
# to avoid possible FPs with (\s+(?==\?))?, look for the whole RE
# separated by whitespace.
1 while ($header =~ s/(=\?[\w_-]+\?[bqBQ]\?[^?]+\?=)\s+(=\?[\w_-]+\?[bqBQ]\?[^?]+\?=)/$1$2/g);
- $header =~
- s/=\?([\w_-]+)\?([bqBQ])\?([^?]+)\?=/__decode_header($1, uc($2), $3)/ge;
+ unless ($header =~
+ s/=\?([\w_-]+)\?([bqBQ])\?([^?]+)\?=/__decode_header($1, uc($2), $3)/ge) {
+ $header = _normalize($header);
+ }
return $header;
}
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm Fri Jan 6 15:09:10 2006
@@ -166,6 +166,12 @@
desc => 'The "sa-update" script requires this module to access compressed
update archive files.',
},
+{
+ module => 'Encode::Detect',
+ version => '0.00',
+ desc => 'If this module is installed, SpamAssassin will detect charsets
+ and convert them into Unicode.',
+},
);
###########################################################################
Modified: spamassassin/trunk/t/memory_cycles.t
URL: http://svn.apache.org/viewcvs/spamassassin/trunk/t/memory_cycles.t?rev=366594&r1=366593&r2=366594&view=diff
==============================================================================
--- spamassassin/trunk/t/memory_cycles.t (original)
+++ spamassassin/trunk/t/memory_cycles.t Fri Jan 6 15:09:10 2006
@@ -21,7 +21,7 @@
use SATest; sa_t_init("memory_cycles");
use Test; BEGIN {
- plan tests => (HAVE_DEVEL_CYCLE ? 3 : 0);
+ plan tests => (HAVE_DEVEL_CYCLE ? 4 : 0);
}
unless (HAVE_DEVEL_CYCLE) {
print "# Devel::Cycle module required for this test, skipped\n";
@@ -55,8 +55,12 @@
$status->finish();
ok (check_for_cycles($status));
+$mail->finish();
+ok (check_for_cycles($mail));
+
$spamtest->finish();
ok (check_for_cycles($spamtest));
+
exit;
############################################################################