You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/09/10 17:59:05 UTC
[incubator-ponymail-foal] branch master updated (32c7cf8 -> bc7bf3f)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from 32c7cf8 trim away the _* fields from what we return in the UI
new 0dbce9d fix header issue, expand to trim for external UI use
new bc7bf3f trim emails before sending payload, to reduce download size
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/endpoints/stats.py | 5 +++++
server/plugins/mbox.py | 26 +++++++++++++++++++++++---
2 files changed, 28 insertions(+), 3 deletions(-)
[incubator-ponymail-foal] 01/02: fix header issue,
expand to trim for external UI use
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 0dbce9d8d0e7e07aaf9039cc383ae4ed186812d2
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 19:58:31 2020 +0200
fix header issue, expand to trim for external UI use
---
server/plugins/mbox.py | 26 +++++++++++++++++++++++---
1 file changed, 23 insertions(+), 3 deletions(-)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index 87f6fdb..dcfb066 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -41,13 +41,32 @@ PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
mbox_cache_privacy: typing.Dict[str, bool] = {}
-
-def trim_email(doc):
+used_ui_fields = [
+ "private",
+ "list",
+ "attachments",
+ "from",
+ "message-id",
+ "mid",
+ "body",
+ "epoch",
+ "subject",
+ "id",
+ "gravatar"
+]
+
+
+def trim_email(doc, external=False):
"""Trims away document fields not used by the UI"""
- for header in doc.keys():
+ for header in list(doc.keys()):
+ # Remove meta data fields which start with an underscore
if header.startswith('_'):
del doc[header]
+ # Remove other fields not used by the UI, if for external consumption
+ elif external and header not in used_ui_fields:
+ del doc[header]
+
def extract_name(addr):
""" Extract name and email from from: header """
@@ -481,3 +500,4 @@ def gravatar(eml):
mailaddr = email.utils.parseaddr(header_from)[1]
ghash = hashlib.md5(mailaddr.encode("utf-8")).hexdigest()
return ghash
+
[incubator-ponymail-foal] 02/02: trim emails before sending payload,
to reduce download size
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit bc7bf3f35cc40b2197b2e039154ea209d93c399d
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 19:58:52 2020 +0200
trim emails before sending payload, to reduce download size
---
server/endpoints/stats.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py
index d142aec..8ac7a02 100644
--- a/server/endpoints/stats.py
+++ b/server/endpoints/stats.py
@@ -47,6 +47,7 @@ async def process(
for msg in results:
msg["gravatar"] = plugins.mbox.gravatar(msg)
+
wordcloud = None
if server.config.ui.wordcloud:
wordcloud = await plugins.mbox.wordcloud(session, query_defuzzed)
@@ -72,6 +73,10 @@ async def process(
}
)
+ # Trim email data so as to reduce download sizes
+ for msg in results:
+ plugins.mbox.trim_email(msg, external=True)
+
return {
"firstYear": first_year,
"lastYear": last_year,