You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by mm...@apache.org on 2015/01/29 18:24:48 UTC
svn commit: r1655758 - in /spamassassin/trunk/lib/Mail/SpamAssassin: Conf.pm
Message/Node.pm Util/DependencyInfo.pm
Author: mmartinec
Date: Thu Jan 29 17:24:47 2015
New Revision: 1655758
URL: http://svn.apache.org/r1655758
Log:
Bug 7126: Incorrect character set detections by normalize_charset
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm?rev=1655758&r1=1655757&r2=1655758&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Thu Jan 29 17:24:47 2015
@@ -1112,9 +1112,21 @@ Select the locales to allow from the lis
=item normalize_charset ( 0 | 1) (default: 0)
-Whether to detect character sets and normalize message content to
-Unicode. Requires the Encode::Detect module, HTML::Parser version
-3.46 or later, and Perl 5.8.5 or later.
+Whether to decode non- UTF-8 and non-ASCII textual parts and recode them
+to UTF-8 before the text is given over to rules processing. The character
+set used for attempted decoding is primarily based on a declared character
+set in a Content-Type header, but if the decoding attempt fails a module
+Encode::Detect::Detector is consulted (if available) to provide a guess
+based on the actual text, and decoding is re-attempted. Even if the option
+is enabled no unnecessary decoding and re-encoding work is done when
+possible (like with an all-ASCII text with a US-ASCII or extended ASCII
+character set declaration, e.g. UTF-8 or ISO-8859-nn or Windows-nnnn).
+
+Unicode support in old versions of perl or in a core module Encode is likely
+to be buggy in places, so if the normalize_charset function is enabled
+it is advised to stick to more recent versions of perl (preferably 5.12
+or later). The module Encode::Detect::Detector is optional, when necessary
+it will be used if it is available.
=cut
@@ -1140,10 +1152,10 @@ Unicode. Requires the Encode::Detect mo
$self->{parser}->lint_warn("config: normalize_charset requires HTML::Parser 3.46 or later");
return $INVALID_VALUE;
}
- unless (eval 'require Encode::Detect::Detector') {
- $self->{parser}->lint_warn("config: normalize_charset requires Encode::Detect");
- return $INVALID_VALUE;
- }
+# unless (eval 'require Encode::Detect::Detector') {
+# $self->{parser}->lint_warn("config: normalize_charset requires Encode::Detect::Detector");
+# return $INVALID_VALUE;
+# }
unless (eval 'require Encode') {
$self->{parser}->lint_warn("config: normalize_charset requires Encode");
return $INVALID_VALUE;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm?rev=1655758&r1=1655757&r2=1655758&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm Thu Jan 29 17:24:47 2015
@@ -43,6 +43,12 @@ use Mail::SpamAssassin::Constants qw(:sa
use Mail::SpamAssassin::HTML;
use Mail::SpamAssassin::Logger;
+our($enc_utf8, $have_encode_detector);
+BEGIN {
+ eval { require Encode && ($enc_utf8 = Encode::find_encoding('UTF-8')) };
+ eval { require Encode::Detect::Detector && ($have_encode_detector = 1) };
+};
+
=item new()
Generates an empty Node object and returns it. Typically only called
@@ -386,28 +392,110 @@ sub _html_render {
}
sub _normalize {
- my ($self, $data, $charset) = @_;
- return $data unless $self->{normalize};
-
- my $detected = Encode::Detect::Detector::detect($data);
-
- my $converter;
-
- if ($charset && $charset !~ /^us-ascii$/i &&
- ($detected || 'none') !~ /^(?:UTF|EUC|ISO-2022|Shift_JIS|Big5|GB)/i) {
- dbg("message: Using labeled charset $charset");
- $converter = Encode::find_encoding($charset);
+ my $self = $_[0];
+# my $data = $_[1]; # avoid copying large strings
+ my $charset_declared = $_[2];
+
+ return $_[1] unless $self->{normalize} && $enc_utf8;
+
+ if (!defined $charset_declared || $charset_declared eq '') {
+ $charset_declared = 'us-ascii';
+ }
+
+ if ($charset_declared =~ /^(?:US-)?ASCII\z/i && $_[1] !~ tr/\x00-\x7F//c ) {
+ # declared as US-ASCII and it really is
+ dbg("message: kept, charset really is US-ASCII as declared");
+ return $_[1]; # is all-ASCII, no need for decoding
+ }
+
+ if ($charset_declared =~
+ /^(?: ISO-?8859-\d{1,2} | Windows-\d{4} | KOI8-[A-Z]{1,2} )\z/xsi &&
+ $_[1] !~ tr/\x00-\x7F//c ) # is all-ASCII
+ { # declared as extended ASCII, but it is actually a plain 7-bit US-ASCII
+ dbg("message: kept, is all-ASCII, declared charset %s", $charset_declared);
+ return $_[1]; # is all-ASCII, no need for decoding
+ }
+
+ # try first to strictly decode based on a declared character set
+
+ my $rv;
+ if ($charset_declared =~ /^(?:US-)?ASCII\z/i) {
+ # declared as US-ASCII but contains 8-bit characters, makes no sense
+ # to attempt decoding first as strict US-ASCII as we know it would fail
+
+ } elsif ($charset_declared =~ /^UTF-?8\z/i) {
+ # attempt decoding as strict UTF-8, but as our intention is to return
+ # the string encoded as UTF-8 anyway, the re-encoding step can be avoided
+ # by returning original string as long as the decoding proves successful
+ if (eval { defined $enc_utf8->decode($_[1], 1|8) }) { # FB_CROAK | LEAVE_SRC
+ dbg("message: kept, valid charset UTF-8");
+ return $_[1];
+ } else {
+ dbg("message: failed decoding as declared charset UTF-8");
+ };
+
+ } else {
+ # try decoding as a declared character set
+
+ # -> http://en.wikipedia.org/wiki/Windows-1252
+ # Windows-1252 character encoding is a superset of ISO 8859-1, but differs
+ # from the IANA's ISO-8859-1 by using displayable characters rather than
+ # control characters in the 80 to 9F (hex) range. [...]
+ # It is very common to mislabel Windows-1252 text with the charset label
+ # ISO-8859-1. A common result was that all the quotes and apostrophes
+ # (produced by "smart quotes" in word-processing software) were replaced
+ # with question marks or boxes on non-Windows operating systems, making
+ # text difficult to read. Most modern web browsers and e-mail clients
+ # treat the MIME charset ISO-8859-1 as Windows-1252 to accommodate
+ # such mislabeling. This is now standard behavior in the draft HTML 5
+ # specification, which requires that documents advertised as ISO-8859-1
+ # actually be parsed with the Windows-1252 encoding.
+ #
+ my $chset = $charset_declared;
+ $chset = 'Windows-1252' if $charset_declared =~ /^ISO-?8859-1\z/i;
+
+ my $decoder = Encode::find_encoding($chset);
+ if ($decoder) {
+ eval { $rv = $decoder->decode($_[1], 1|8) }; # FB_CROAK | LEAVE_SRC
+ if ($chset eq $charset_declared) {
+ dbg("message: %s as declared charset %s",
+ defined $rv ? 'decoded' : 'failed decoding', $charset_declared);
+ } else {
+ dbg("message: %s as charset %s, declared %s",
+ defined $rv ? 'decoded' : 'failed decoding',
+ $chset, $charset_declared);
+ }
+ }
}
- $converter = Encode::find_encoding($detected) unless $converter || !defined($detected);
+ # if we were unsuccessful so far, try some guesswork
+ # based on Encode::Detect::Detector
- return $data unless $converter;
+ if (defined $rv) {
+ # done, no need for guesswork
+ } elsif (!$have_encode_detector) {
+ dbg("message: Encode::Detect::Detector not available, declared %s failed",
+ $charset_declared);
+ } else {
+ my $charset_detected = Encode::Detect::Detector::detect($_[1]);
+ if ($charset_detected && lc $charset_detected ne lc $charset_declared) {
+ my $decoder = Encode::find_encoding($charset_detected);
+ if ($decoder) {
+ eval { $rv = $decoder->decode($_[1], 1|8) }; # FB_CROAK | LEAVE_SRC
+ dbg("message: %s as detected charset %s, declared %s",
+ defined $rv ? 'decoded' : 'failed decoding',
+ $charset_detected, $charset_declared);
+ }
+ }
+ }
- dbg("message: Converting...");
+ if (!defined $rv) {
+ # decoding attempts failed, return unchanged octets
+ return $_[1]; # garbage-in / garbage-out
+ }
- my $rv = $converter->decode($data, 0);
- utf8::downgrade($rv, 1);
- return $rv
+ # decoding octets to characters was successful, now encode it as UTF-8 octets
+ return $enc_utf8->encode($rv);
}
=item rendered()
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
URL: http://svn.apache.org/viewvc/spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm?rev=1655758&r1=1655757&r2=1655758&view=diff
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm (original)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm Thu Jan 29 17:24:47 2015
@@ -241,11 +241,13 @@ $have_sha ? {
If-Modified-Since GET requests.',
},
{
- module => 'Encode::Detect',
+ module => 'Encode::Detect::Detector',
version => 0,
- desc => 'If you plan to use the normalize_charset config setting to detect
- charsets and convert them into Unicode, you will need to install
- this module.',
+ desc => 'If you plan to use the normalize_charset config setting to
+ decode message parts from their declared character set into Unicode, and
+ such decoding fails, the Encode::Detect::Detector module (when available)
+ may be consulted to provide an alternative guess on a character set of a
+ problematic message part.',
},
{
module => 'Net::Patricia',