You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/09/10 15:00:22 UTC
[incubator-ponymail-foal] branch master updated (5bf34d0 -> 32c7cf8)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from 5bf34d0 note what we imported it as
new 9a35648 rename notes to _notes, add an additional _archived_at field
new 1d0ed5b this field isn't used
new 32c7cf8 trim away the _* fields from what we return in the UI
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/plugins/mbox.py | 10 ++++++++++
tools/archiver.py | 3 ++-
tools/import-mbox.py | 4 ++--
tools/mappings.yaml | 7 +++----
tools/migrate.py | 4 ++--
5 files changed, 19 insertions(+), 9 deletions(-)
[incubator-ponymail-foal] 03/03: trim away the _* fields from what
we return in the UI
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 32c7cf841a0587e6f9c53e2ead11b4f34e4bb83c
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 17:00:00 2020 +0200
trim away the _* fields from what we return in the UI
We don't need the meta fields for UI operations.
---
server/plugins/mbox.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index 338d82f..87f6fdb 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -42,6 +42,13 @@ PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
mbox_cache_privacy: typing.Dict[str, bool] = {}
+def trim_email(doc):
+ """Trims away document fields not used by the UI"""
+ for header in doc.keys():
+ if header.startswith('_'):
+ del doc[header]
+
+
def extract_name(addr):
""" Extract name and email from from: header """
m = re.match(r"^([^<]+)\s*<(.+)>$", addr)
@@ -196,6 +203,7 @@ async def get_email(
if doc and plugins.aaa.can_access_email(session, doc):
if not session.credentials:
doc = anonymize(doc)
+ trim_email(doc)
return doc
# multi-doc return?
@@ -207,6 +215,7 @@ async def get_email(
if doc and plugins.aaa.can_access_email(session, doc):
if not session.credentials:
doc = anonymize(doc)
+ trim_email(doc)
docs_returned.append(doc)
return docs_returned
# no doc?
@@ -289,6 +298,7 @@ async def query(
doc = anonymize(doc)
if shorten:
doc["body"] = (doc["body"] or "")[:200]
+ trim_email(doc)
docs.append(doc)
hits += 1
if hits > query_limit:
[incubator-ponymail-foal] 02/03: this field isn't used
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 1d0ed5b71ce4d0d55ef2c23b665a64578d076f85
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 16:52:33 2020 +0200
this field isn't used
---
tools/mappings.yaml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index f0fcfdf..10984b8 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -37,9 +37,6 @@ mailinglist:
type: keyword
mbox:
properties:
- '@import_timestamp':
- format: yyyy/MM/dd HH:mm:ss||yyyy/MM/dd
- type: date
attachments:
properties:
content_type:
[incubator-ponymail-foal] 01/03: rename notes to _notes,
add an additional _archived_at field
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 9a356488d8b6cceebbfb11a4994a09a9a09320a6
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 16:51:22 2020 +0200
rename notes to _notes, add an additional _archived_at field
---
tools/archiver.py | 3 ++-
tools/import-mbox.py | 4 ++--
tools/mappings.yaml | 4 +++-
tools/migrate.py | 4 ++--
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/tools/archiver.py b/tools/archiver.py
index 255866f..a6d236b 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -494,7 +494,8 @@ class Archiver(object): # N.B. Also used by import-mbox.py
"body": body.unflow() if body else "",
"html_source_only": body and body.html_as_source or False,
"attachments": attachments,
- "notes": notes,
+ "_notes": notes,
+ "_archived_at": int(time.time()),
}
return output_json, contents, msg_metadata, irt
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 02ccf55..21ca9ba 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -311,8 +311,8 @@ class SlurpThread(Thread):
]
# Mark that we imported this email
- json["notes"] = [x for x in json["notes"] if "ARCHIVE:" not in x] # Pop archiver.py note
- json["notes"].append(["IMPORT: Email imported as %s at %u" % (json["mid"], time.time())])
+ json["_notes"] = [x for x in json["_notes"] if "ARCHIVE:" not in x] # Pop archiver.py note
+ json["_notes"].append(["IMPORT: Email imported as %s at %u" % (json["mid"], time.time())])
try: # temporary hack to try and find an encoding issue
# needs to be replaced by proper exception handling
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index 8c881b7..f0fcfdf 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -87,8 +87,10 @@ mbox:
type: text
to:
type: text
- notes:
+ _notes:
type: text
+ _archived_at:
+ type: long
notification:
properties:
date:
diff --git a/tools/migrate.py b/tools/migrate.py
index 3968709..74c14f5 100644
--- a/tools/migrate.py
+++ b/tools/migrate.py
@@ -109,7 +109,7 @@ async def main():
doc['_source']['dbid'] = hashlib.sha3_256(source_text).hexdigest()
# Append migration details to notes field in doc
- notes = doc['_source'].get('notes', [])
+ notes = doc['_source'].get('_notes', [])
# We want a list, not a single string
if isinstance(notes, str):
notes = list(notes)
@@ -119,7 +119,7 @@ async def main():
if do_dkim:
notes.append("REINDEX: Document re-indexed with DKIM_ID at %u, "
"from %s to %s" % (now, dkim_id, old_id))
- doc['_source']['notes'] = notes
+ doc['_source']['_notes'] = notes
# Copy to new DB
bulk_array.append({