[allura] 01/02: [#8289] fix inbound mail encoding issue

[br...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

brondsem pushed a commit to branch db/8289
in repository https://gitbox.apache.org/repos/asf/allura.git

commit 553a3f7130f2ff0151dafae07133545add8d265c
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Thu May 30 13:50:48 2019 -0400

    [#8289] fix inbound mail encoding issue
---
 Allura/allura/lib/mail_util.py        |  5 +++-
 Allura/allura/tasks/mail_tasks.py     |  2 +-
 Allura/allura/tests/test_mail_util.py | 54 ++++++++++++++++++++++++++++++++++-
 3 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/Allura/allura/lib/mail_util.py b/Allura/allura/lib/mail_util.py
index ea906bf..7fbace5 100644
--- a/Allura/allura/lib/mail_util.py
+++ b/Allura/allura/lib/mail_util.py
@@ -23,6 +23,7 @@ from email.MIMEMultipart import MIMEMultipart
 from email.MIMEText import MIMEText
 from email import header
 
+import six
 import tg
 from paste.deploy.converters import asbool, asint, aslist
 from formencode import validators as fev
@@ -155,8 +156,10 @@ def parse_message(data):
     else:
         result['payload'] = msg.get_payload(decode=True)
         charset = msg.get_content_charset()
-        if charset:
+        # payload is sometimes already unicode (due to being saved in mongo?)
+        if isinstance(result['payload'], six.binary_type) and charset:
             result['payload'] = result['payload'].decode(charset)
+
     return result
 
 
diff --git a/Allura/allura/tasks/mail_tasks.py b/Allura/allura/tasks/mail_tasks.py
index d60533f..43a9f1a 100644
--- a/Allura/allura/tasks/mail_tasks.py
+++ b/Allura/allura/tasks/mail_tasks.py
@@ -62,7 +62,7 @@ def route_email(
     '''
     try:
         msg = mail_util.parse_message(data)
-    except:  # pragma no cover
+    except Exception:  # pragma no cover
         log.exception('Parse Error: (%r,%r,%r)', peer, mailfrom, rcpttos)
         return
     if mail_util.is_autoreply(msg):
diff --git a/Allura/allura/tests/test_mail_util.py b/Allura/allura/tests/test_mail_util.py
index 13b5b42..4f99840 100644
--- a/Allura/allura/tests/test_mail_util.py
+++ b/Allura/allura/tests/test_mail_util.py
@@ -22,7 +22,7 @@ from email.MIMEMultipart import MIMEMultipart
 from email.MIMEText import MIMEText
 
 import mock
-from nose.tools import raises, assert_equal, assert_false, assert_true
+from nose.tools import raises, assert_equal, assert_false, assert_true, assert_in
 from ming.orm import ThreadLocalORMSession
 from tg import config as tg_config
 
@@ -96,6 +96,58 @@ class TestReactor(unittest.TestCase):
         s_msg = msg1.as_string()
         msg2 = parse_message(s_msg)
         assert isinstance(msg2['payload'], unicode)
+        assert_in(u'всех', msg2['payload'])
+
+    def test_more_encodings(self):
+        # these are unicode strings to reflect behavior after loading 'route_email' tasks from mongo
+        s_msg = u"""Date: Sat, 25 May 2019 09:32:00 +1000
+From: <fo...@bar.com>
+To: <38...@bugs.proj.localhost>
+Subject: bugs
+Content-Type: text/plain; charset=GBK
+Content-Transfer-Encoding: base64
+
+VGhlIFNuYXA3IGFwcGxpY2F0aW9uKGJhc2VkIG9uIHNuYXA3LWZ1bGwtMS40LjIpIGhhcyBiZWVu
+IHJ1biBvdmVyIGEgd2VlayBvbiBRTlg2LjYuMCwKQnV0IHNvbWV0aW1lcyAsc3lzdGVtIHNjcmVl
+biB3aWxsIHByaW50CiJsZGQ6RkFUQUw6Y291bGQgbm90IGxvYWQgbGlicmFyeSBsaWJzb2NrZXQu
+c28uMyIsClRoZSBhcHBsaWNhdGlvbidzIGNvbW11bmljYXRpb24gd29yayB3ZWxsICxidXQgdGhl
+IGZ0cCx0ZWxuZXQscGluZyBjYW4ndCB3b3JrICEKCgpXaHk/
+"""
+        msg = parse_message(s_msg)
+        assert isinstance(msg['payload'], unicode)
+        assert_in(u'The Snap7 application', msg['payload'])
+
+        s_msg = u"""Date: Sat, 25 May 2019 09:32:00 +1000
+From: <fo...@bar.com>
+To: <38...@bugs.proj.localhost>
+Subject: bugs
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+> Status: closed
+> Created: Thu May 23, 2019 09:24 PM UTC by admin1
+> Attachments:
+> 
+>   • foo.txt (1.0 kB; text/plain)
+> 
+"""
+        msg = parse_message(s_msg)
+        assert isinstance(msg['payload'], unicode)
+        assert_in(u'• foo', msg['payload'])
+
+        s_msg = u"""Date: Sat, 25 May 2019 09:32:00 +1000
+From: <fo...@bar.com>
+To: <38...@bugs.proj.localhost>
+Subject: bugs
+Content-Type: TEXT/PLAIN; format=flowed; charset=ISO-8859-15
+Content-Transfer-Encoding: 8BIT
+
+programmed or èrogrammed ?
+"""
+        msg = parse_message(s_msg)
+        assert isinstance(msg['payload'], unicode)
+        assert_in(u'èrogrammed', msg['payload'])
 
     def test_unicode_complex_message(self):
         charset = 'utf-8'