You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/08/12 07:56:28 UTC
[incubator-ponymail-unit-tests] branch master updated (2f142f4 ->
28055ac)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git.
from 2f142f4 cli args for runall.py
new 3b6676b allow for generating from multiple mbox files, be more lenient with email bodies
new 28055ac Miscellaneous parsing tests and corpus
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
corpus/empty_attachment.mbox | 44 +++++++++++++++++++++
corpus/empty_body.mbox | 51 ++++++++++++++++++++++++
corpus/no_date.mbox | 5 +++
corpus/text_enriched.mbox | 87 +++++++++++++++++++++++++++++++++++++++++
tests/test-parsing.py | 40 +++++++++++--------
yaml/parsing-miscellaneous.yaml | 26 ++++++++++++
6 files changed, 237 insertions(+), 16 deletions(-)
create mode 100644 corpus/empty_attachment.mbox
create mode 100644 corpus/empty_body.mbox
create mode 100644 corpus/no_date.mbox
create mode 100644 corpus/text_enriched.mbox
create mode 100644 yaml/parsing-miscellaneous.yaml
[incubator-ponymail-unit-tests] 01/02: allow for generating from
multiple mbox files, be more lenient with email bodies
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
commit 3b6676b4a1b309b4b315a0b6382c87f95076f7ce
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Wed Aug 12 09:55:45 2020 +0200
allow for generating from multiple mbox files, be more lenient with email bodies
---
tests/test-parsing.py | 40 ++++++++++++++++++++++++----------------
1 file changed, 24 insertions(+), 16 deletions(-)
diff --git a/tests/test-parsing.py b/tests/test-parsing.py
index 4937fad..cf14bcf 100644
--- a/tests/test-parsing.py
+++ b/tests/test-parsing.py
@@ -20,21 +20,27 @@ def generate_specs(args):
import archiver
archie = archiver.Archiver(parse_html=args.html)
sys.stderr.write("Generating parsing specs for file '%s'...\n" % args.mboxfile)
- gen_spec = []
- mbox = mailbox.mbox(args.mboxfile, None, create=False)
- for key in mbox.keys():
- message_raw = mbox.get_bytes(key) # True raw format, as opposed to calling .as_bytes()
- message = mbox.get(key)
- lid = archiver.normalize_lid(message.get('list-id', '??'))
- json, _, _, _ = archie.compute_updates(fake_args, lid, False, message, message_raw)
- gen_spec.append({
- 'index': key,
- 'message-id': message.get('message-id').strip(),
- 'body_sha3_256': hashlib.sha3_256(json['body'].encode('utf-8')).hexdigest() if json and json['body'] else None,
- 'attachments': json['attachments'] if json else None,
- })
+ items = {}
+ for mboxfile in args.mboxfile:
+ tests = []
+ mbox = mailbox.mbox(mboxfile, None, create=False)
+ for key in mbox.keys():
+ message_raw = mbox.get_bytes(key) # True raw format, as opposed to calling .as_bytes()
+ message = mbox.get(key)
+ lid = archiver.normalize_lid(message.get('list-id', '??'))
+ json, _, _, _ = archie.compute_updates(fake_args, lid, False, message, message_raw)
+ body_sha3_256 = None
+ if json and json.get('body') is not None:
+ body_sha3_256 = hashlib.sha3_256(json['body'].encode('utf-8')).hexdigest()
+ tests.append({
+ 'index': key,
+ 'message-id': message.get('message-id', '').strip(),
+ 'body_sha3_256': body_sha3_256,
+ 'attachments': json['attachments'] if json else None,
+ })
+ items[mboxfile] = tests
with open(args.generate, 'w') as f:
- yaml.dump({'args': {'cmd': " ".join(sys.argv), 'parse_html': True if args.html else False}, 'parsing': {args.mboxfile: gen_spec}}, f)
+ yaml.dump({'args': {'cmd': " ".join(sys.argv), 'parse_html': True if args.html else False}, 'parsing': items}, f)
f.close()
@@ -58,7 +64,9 @@ def run_tests(args):
message = mbox.get(test['index'])
lid = archiver.normalize_lid(message.get('list-id', '??'))
json, _, _, _ = archie.compute_updates(fake_args, lid, False, message, message_raw)
- body_sha3_256 = hashlib.sha3_256(json['body'].encode('utf-8')).hexdigest() if json and json['body'] else None
+ body_sha3_256 = None
+ if json and json.get('body') is not None:
+ body_sha3_256 = hashlib.sha3_256(json['body'].encode('utf-8')).hexdigest()
if body_sha3_256 != test['body_sha3_256']:
errors += 1
sys.stderr.write("""[FAIL] index %u: \nExpected:\n%s\nGot:\n%s\n""" %
@@ -80,7 +88,7 @@ def main():
help='Generate a test yaml spec, output to file specified here')
parser.add_argument('--load', dest='load', type=str,
help='Load and run tests from a yaml spec file')
- parser.add_argument('--mbox', dest='mboxfile', type=str,
+ parser.add_argument('--mbox', dest='mboxfile', type=str, nargs='+',
help='If generating spec, which mbox corpus file to use for testing')
parser.add_argument('--rootdir', dest='rootdir', type=str, required=True,
help="Root directory of Apache Pony Mail")
[incubator-ponymail-unit-tests] 02/02: Miscellaneous parsing tests
and corpus
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
commit 28055acf2b2a5fae51578fb64320a65d6fc2b9a6
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Wed Aug 12 09:56:11 2020 +0200
Miscellaneous parsing tests and corpus
---
corpus/empty_attachment.mbox | 44 +++++++++++++++++++++
corpus/empty_body.mbox | 51 ++++++++++++++++++++++++
corpus/no_date.mbox | 5 +++
corpus/text_enriched.mbox | 87 +++++++++++++++++++++++++++++++++++++++++
yaml/parsing-miscellaneous.yaml | 26 ++++++++++++
5 files changed, 213 insertions(+)
diff --git a/corpus/empty_attachment.mbox b/corpus/empty_attachment.mbox
new file mode 100644
index 0000000..07b490f
--- /dev/null
+++ b/corpus/empty_attachment.mbox
@@ -0,0 +1,44 @@
+From tomcat-user-return-50118-qmlist-jakarta-archive-tomcat-user=jakarta.apache.org@jakarta.apache.org Fri Jan 24 12:24:14 2003
+Return-Path: <to...@jakarta.apache.org>
+Delivered-To: apmail-jakarta-tomcat-user-archive@apache.org
+Received: (qmail 13295 invoked from network); 24 Jan 2003 12:24:14 -0000
+Received: from exchange.sun.com (192.18.33.10)
+ by 208.185.179.12.available.above.net with SMTP; 24 Jan 2003 12:24:14 -0000
+Received: (qmail 24215 invoked by uid 97); 24 Jan 2003 12:25:16 -0000
+Delivered-To: qmlist-jakarta-archive-tomcat-user@jakarta.apache.org
+Received: (qmail 24199 invoked by uid 97); 24 Jan 2003 12:25:16 -0000
+Mailing-List: contact tomcat-user-help@jakarta.apache.org; run by ezmlm
+Precedence: bulk
+List-Unsubscribe: <ma...@jakarta.apache.org>
+List-Subscribe: <ma...@jakarta.apache.org>
+List-Help: <ma...@jakarta.apache.org>
+List-Post: <ma...@jakarta.apache.org>
+List-Id: "Tomcat Users List" <tomcat-user.jakarta.apache.org>
+Reply-To: "Tomcat Users List" <to...@jakarta.apache.org>
+Delivered-To: mailing list tomcat-user@jakarta.apache.org
+Received: (qmail 24187 invoked by uid 98); 24 Jan 2003 12:25:15 -0000
+X-Antivirus: nagoya (v4218 created Aug 14 2002)
+Message-ID: <00...@win98>
+From: "Henry" <he...@cafelab.com>
+To: "Tomcat Users List" <to...@jakarta.apache.org>
+Subject: how do I detect alive sessions at this moment?
+Date: Fri, 24 Jan 2003 20:24:15 +0800
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="----=_NextPart_000_003F_01C2C3E6.90B6F900"
+X-Priority: 3
+X-MSMail-Priority: Normal
+X-Mailer: Microsoft Outlook Express 6.00.2800.1106
+X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2800.1106
+X-Spam-Rating: 208.185.179.12.available.above.net 1.6.2 0/1000/N
+X-Spam-Rating: 208.185.179.12.available.above.net 1.6.2 0/1000/N
+
+------=_NextPart_000_003F_01C2C3E6.90B6F900
+Content-Type: text/plain;
+ charset="big5"
+Content-Transfer-Encoding: quoted-printable
+
+
+------=_NextPart_000_003F_01C2C3E6.90B6F900--
+
+
diff --git a/corpus/empty_body.mbox b/corpus/empty_body.mbox
new file mode 100644
index 0000000..d1f849b
--- /dev/null
+++ b/corpus/empty_body.mbox
@@ -0,0 +1,51 @@
+From issues-return-129-apmail-ponymail-issues-archive=ponymail.apache.org@ponymail.incubator.apache.org Thu Nov 17 00:49:30 2016
+Return-Path: <is...@ponymail.incubator.apache.org>
+X-Original-To: apmail-ponymail-issues-archive@minotaur.apache.org
+Delivered-To: apmail-ponymail-issues-archive@minotaur.apache.org
+Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
+ by minotaur.apache.org (Postfix) with SMTP id A72D919611
+ for <ap...@minotaur.apache.org>; Thu, 17 Nov 2016 00:49:30 +0000 (UTC)
+Received: (qmail 22868 invoked by uid 500); 17 Nov 2016 00:49:30 -0000
+Delivered-To: apmail-ponymail-issues-archive@ponymail.apache.org
+Received: (qmail 22841 invoked by uid 500); 17 Nov 2016 00:49:30 -0000
+Mailing-List: contact issues-help@ponymail.incubator.apache.org; run by ezmlm
+Precedence: bulk
+List-Help: <ma...@ponymail.incubator.apache.org>
+List-Unsubscribe: <ma...@ponymail.incubator.apache.org>
+List-Post: <ma...@ponymail.incubator.apache.org>
+List-Id: <issues.ponymail.incubator.apache.org>
+Reply-To: dev@ponymail.incubator.apache.org
+Delivered-To: mailing list issues@ponymail.incubator.apache.org
+Received: (qmail 22832 invoked by uid 99); 17 Nov 2016 00:49:30 -0000
+Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142)
+ by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 17 Nov 2016 00:49:30 +0000
+Received: from localhost (localhost [127.0.0.1])
+ by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 2614B1A00A2
+ for <is...@ponymail.apache.org>; Thu, 17 Nov 2016 00:49:30 +0000 (UTC)
+X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org
+X-Spam-Flag: NO
+X-Spam-Score: -7.019
+X-Spam-Level:
+X-Spam-Status: No, score=-7.019 tagged_above=-999 required=6.31
+ tests=[KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5,
+ RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01,
+ RP_MATCHES_RCVD=-2.999] autolearn=disabled
+Received: from mx1-lw-eu.apache.org ([10.40.0.8])
+ by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024)
+ with ESMTP id XffM2rPGCqQk for <is...@ponymail.apache.org>;
+ Thu, 17 Nov 2016 00:49:29 +0000 (UTC)
+Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
+ by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id DD1285FCC9
+ for <is...@ponymail.incubator.apache.org>; Thu, 17 Nov 2016 00:49:28 +0000 (UTC)
+Received: (qmail 22820 invoked by uid 99); 17 Nov 2016 00:49:28 -0000
+Received: from minotaur.apache.org (HELO minotaur.apache.org) (140.211.11.9)
+ by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 17 Nov 2016 00:49:28 +0000
+Received: by minotaur.apache.org (Postfix, from userid 1721)
+ id D69A719610; Thu, 17 Nov 2016 00:49:27 +0000 (UTC)
+To: issues@ponymail.incubator.apache.org
+Subject: Test email with empty body
+Message-Id: <20...@minotaur.apache.org>
+Date: Thu, 17 Nov 2016 00:49:27 +0000 (UTC)
+From: sebb@apache.org (Sebastian Bazley)
+
+
diff --git a/corpus/no_date.mbox b/corpus/no_date.mbox
new file mode 100644
index 0000000..220af97
--- /dev/null
+++ b/corpus/no_date.mbox
@@ -0,0 +1,5 @@
+From
+List-Id: <te...@test.invalid>
+
+No date
+
diff --git a/corpus/text_enriched.mbox b/corpus/text_enriched.mbox
new file mode 100644
index 0000000..cb4b797
--- /dev/null
+++ b/corpus/text_enriched.mbox
@@ -0,0 +1,87 @@
+From tomcat-user-return-34660-apmail-jakarta-tomcat-user-archive=jakarta.apache.org@jakarta.apache.org Mon May 14 19:23:38 2001
+Return-Path: <to...@jakarta.apache.org>
+Delivered-To: apmail-jakarta-tomcat-user-archive@jakarta.apache.org
+Received: (qmail 71623 invoked by uid 500); 14 May 2001 19:23:25 -0000
+Mailing-List: contact tomcat-user-help@jakarta.apache.org; run by ezmlm
+Precedence: bulk
+Reply-To: tomcat-user@jakarta.apache.org
+list-help: <ma...@jakarta.apache.org>
+list-unsubscribe: <ma...@jakarta.apache.org>
+list-post: <ma...@jakarta.apache.org>
+List-Id: <tomcat-user.jakarta.apache.org>
+Delivered-To: mailing list tomcat-user@jakarta.apache.org
+Received: (qmail 71599 invoked from network); 14 May 2001 19:23:07 -0000
+Received: from netgate.emmes.com (root@208.208.19.150)
+ by h31.sny.collab.net with SMTP; 14 May 2001 19:23:07 -0000
+Received: from kumar ([10.0.0.229])
+ by netgate.emmes.com (8.9.3/8.9.3) with SMTP id OAA09624
+ for <to...@jakarta.apache.org>; Mon, 14 May 2001 14:58:19 -0400
+Message-Id: <3....@netgate.emmes.com>
+X-Sender: kumar@netgate.emmes.com
+X-Mailer: QUALCOMM Windows Eudora Pro Version 3.0.5 (32)
+Date: Mon, 14 May 2001 15:23:21 -0400
+To: tomcat-user@jakarta.apache.org
+From: Kumar Thotapally <kt...@emmes.com>
+Subject: Virtual Hosts
+Mime-Version: 1.0
+Content-Type: text/enriched; charset="us-ascii"
+X-Spam-Rating: h31.sny.collab.net 1.6.2 0/1000/N
+
+Hi,
+
+
+I am able to startup and shutdown multiple JVMs (using server1.xml ...
+etc). I created applications (for example, JVM1, JVM2) under web-apps.
+
+
+When I enter http://localhost:8007/jvm1/servlet/HelloWorld1 in my
+browser,
+
+it works fine.
+
+
+However, with the following code in server1.xml :
+
+
+<fontfamily><param>Courier New</param><bigger> <<ContextManager
+debug="0" workDir="work_jvm1" showDebugInfo="true" >
+
+</bigger></fontfamily>
+
+followed by
+
+<fontfamily><param>Courier New</param><bigger> <<!-- @@@ the /jvm1
+context -->
+
+
+ <<Host name="www.tomcat1.com" >
+
+ <<Context path="/jvm1"
+
+ docBase="webapps/jvm1"
+
+ crossContext="false"
+
+</bigger></fontfamily><fontfamily><param>Courier New</param><bigger>
+debug="1"
+
+ reloadable="true" >
+
+ <</Context>
+
+ <</Host>
+
+</bigger></fontfamily>along with other contexts.
+
+
+When I enter www.tomcat1.com, I am not able to get the result.
+
+
+Any suggestions?
+
+
+Thanks,
+
+
+Kumar.
+
diff --git a/yaml/parsing-miscellaneous.yaml b/yaml/parsing-miscellaneous.yaml
new file mode 100644
index 0000000..ec5f522
--- /dev/null
+++ b/yaml/parsing-miscellaneous.yaml
@@ -0,0 +1,26 @@
+args:
+ cmd: tests/test-parsing.py --rootdir /x1/pypony/ --mbox corpus/empty_attachment.mbox
+ corpus/empty_body.mbox corpus/no_date.mbox corpus/text_enriched.mbox --generate
+ yaml/parsing-miscellaneous.yaml
+ parse_html: false
+parsing:
+ corpus/empty_attachment.mbox:
+ - attachments: []
+ body_sha3_256: a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a
+ index: 0
+ message-id: <00...@win98>
+ corpus/empty_body.mbox:
+ - attachments: []
+ body_sha3_256: a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a
+ index: 0
+ message-id: <20...@minotaur.apache.org>
+ corpus/no_date.mbox:
+ - attachments: []
+ body_sha3_256: d8e5444ff55847c68aa94019af9cfc2527597f07a80444ea4ca4a96e6de48bc6
+ index: 0
+ message-id: ''
+ corpus/text_enriched.mbox:
+ - attachments: []
+ body_sha3_256: a8001fbfc118043dc5ddfc4dec185ccd8e6a03505f64adf4ffe5f91ffb66fe48
+ index: 0
+ message-id: <3....@netgate.emmes.com>