You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by se...@apache.org on 2017/09/21 02:19:24 UTC
incubator-ponymail git commit: Archiver loses some message information
Repository: incubator-ponymail
Updated Branches:
refs/heads/master e6daa748a -> d23d649ae
Archiver loses some message information
This fixes #402
Mail parsing can change source spacing
This fixes #410
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/d23d649a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/d23d649a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/d23d649a
Branch: refs/heads/master
Commit: d23d649ae46ecb17d5331571c09d6fa58a9b3859
Parents: e6daa74
Author: Sebb <se...@apache.org>
Authored: Thu Sep 21 03:19:19 2017 +0100
Committer: Sebb <se...@apache.org>
Committed: Thu Sep 21 03:19:19 2017 +0100
----------------------------------------------------------------------
CHANGELOG.md | 2 ++
tools/archiver.py | 11 ++++-------
tools/import-mbox.py | 8 ++++++--
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ecf255..cda0c9c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,6 @@
## CHANGES in 0.10:
+- Archiver loses some message information (#402)
+- Mail parsing can change source spacing (#410)
- Bug: use actual envelope header if present (#412)
- Bug: stats.lua should use UTC for calculating year and month (#409)
- Bug: atom.lua - should generate date in UTC, not local time (#408)
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 3785c86..78db074 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -359,7 +359,7 @@ class Archiver(object):
return ojson, contents
- def archive_message(self, mlist, msg):
+ def archive_message(self, mlist, msg, raw_msg):
"""Send the message to the archiver.
:param mlist: The IMailingList object.
@@ -416,7 +416,7 @@ class Archiver(object):
consistency = self.consistency,
body = {
"message-id": msg_metadata['message-id'],
- "source": self.mbox_source(msg)
+ "source": self.mbox_source(raw_msg)
}
)
@@ -509,11 +509,8 @@ class Archiver(object):
logger.info("Notification sent to %s for %s" % (cid, mid))
return lid, ojson['mid']
- def mbox_source(self, msg):
+ def mbox_source(self, b):
# Common method shared with import-mbox
- policy = msg.policy.clone(max_line_length=0) # don't wrap headers
- hasFrom=(msg.get_unixfrom() != None) # only use the envelope if it exists
- b = msg.as_bytes(unixfrom=hasFrom, policy=policy)
try:
# Can we store as ASCII?
return b.decode('ascii', errors='strict')
@@ -643,7 +640,7 @@ if __name__ == '__main__':
msg_metadata = namedtuple('importmsg', ['list_id', 'archive_public'])(list_id = msg.get('list-id'), archive_public=ispublic)
try:
- lid, mid = archie.archive_message(msg_metadata, msg)
+ lid, mid = archie.archive_message(msg_metadata, msg, msgstring)
print("%s: Done archiving to %s as %s!" % (email.utils.formatdate(), lid, mid))
except Exception as err:
if args.verbose:
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/tools/import-mbox.py
----------------------------------------------------------------------
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index a3b49a0..9eb5bd4 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -239,7 +239,8 @@ class SlurpThread(Thread):
LEY = EY
- for message in messages:
+ for key in messages.iterkeys():
+ message=messages.get(key)
# If --filter is set, discard any messages not matching by continuing to next email
if fromFilter and 'from' in message and message['from'].find(fromFilter) == -1:
continue
@@ -306,6 +307,9 @@ class SlurpThread(Thread):
continue
if json:
+ file=messages.get_file(key, True)
+ raw_msg=file.read()
+ file.close()
if args.dups:
try:
duplicates[json['mid']].append(json['message-id'] + " in " + filename)
@@ -317,7 +321,7 @@ class SlurpThread(Thread):
json_source = {
'mid': json['mid'], # needed for bulk-insert only, not needed in database
'message-id': json['message-id'],
- 'source': archie.mbox_source(message)
+ 'source': archie.mbox_source(raw_msg)
}
except Exception as e:
self.printid("Error '%s' processing id %s msg %s " % (e, json['mid'], json['message-id']))