[incubator-ponymail-foal] branch master updated: Create database id has once to ensure consistency

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git


The following commit(s) were added to refs/heads/master by this push:
     new e0b1c86  Create database id has once to ensure consistency
e0b1c86 is described below

commit e0b1c863349f27da5bc068f9b9a0bf1f309dc642
Author: Sebb <se...@apache.org>
AuthorDate: Fri Sep 4 00:49:54 2020 +0100

    Create database id has once to ensure consistency
---
 tools/archiver.py    | 6 +++---
 tools/import-mbox.py | 3 +--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/archiver.py b/tools/archiver.py
index be6a8b0..d447c62 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -432,6 +432,7 @@ class Archiver(object):  # N.B. Also used by import-mbox.py
                 "subject": msg_metadata["subject"],
                 "message-id": msg_metadata["message-id"],
                 "mid": mid,
+                "dbid": hashlib.sha3_256(raw_msg).hexdigest(),
                 "cc": msg_metadata.get("cc"),
                 "epoch": epoch,
                 "list": lid,
@@ -473,7 +474,6 @@ class Archiver(object):  # N.B. Also used by import-mbox.py
         ojson, contents, msg_metadata, irt = self.compute_updates(
             lid, private, msg, raw_message
         )
-        sha3 = hashlib.sha3_256(raw_message).hexdigest()
         if not ojson:
             _id = msg.get("message-id") or msg.get("Subject") or msg.get("Date")
             raise Exception("Could not parse message %s for %s" % (_id, lid))
@@ -509,7 +509,7 @@ class Archiver(object):  # N.B. Also used by import-mbox.py
 
             elastic.index(
                 index=elastic.db_source,
-                id=sha3,
+                id=ojson["dbid"],
                 body={
                     "message-id": msg_metadata["message-id"],
                     "permalink": ojson["mid"],
@@ -532,7 +532,7 @@ class Archiver(object):  # N.B. Also used by import-mbox.py
                             "id": ojson["mid"],
                             "mbox": ojson,
                             "mbox_source": {
-                                "id": sha3,
+                                "id": ojson["dbid"],
                                 "permalink": ojson["mid"],
                                 "message-id": msg_metadata["message-id"],
                                 "source": mbox_source(raw_message),
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 1155f3a..821749a 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -215,7 +215,6 @@ class SlurpThread(Thread):
                     file = MboxoReader(file)
                 message_raw = file.read()
                 file.close()
-                sha3 = hashlib.sha3_256(message_raw).hexdigest()
                 # If --filter is set, discard any messages not matching by continuing to next email
                 if (
                     fromFilter
@@ -315,7 +314,7 @@ class SlurpThread(Thread):
                         # needs to be replaced by proper exception handling
                         json_source = {
                             "permalink": json["mid"],
-                            "mid": sha3,
+                            "mid": json["dbid"],
                             "message-id": json["message-id"],
                             "source": archiver.mbox_source(message_raw),
                         }