You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by se...@apache.org on 2021/11/22 22:51:28 UTC
[incubator-ponymail-foal] branch master updated (9ded650 -> 6cb859d)
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from 9ded650 fix typo
new ddb0af0 Don't fetch fields that aren't needed
new 6cb859d Update revision
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/endpoints/stats.py | 14 +++++++++-----
server/main.py | 2 +-
server/plugins/messages.py | 12 ++++++++----
3 files changed, 18 insertions(+), 10 deletions(-)
[incubator-ponymail-foal] 02/02: Update revision
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 6cb859d0dd26599f0a0f0f0b7805cadcaca0c931
Author: Sebb <se...@apache.org>
AuthorDate: Mon Nov 22 22:51:16 2021 +0000
Update revision
---
server/main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/main.py b/server/main.py
index 003e191..e70a5ef 100644
--- a/server/main.py
+++ b/server/main.py
@@ -40,7 +40,7 @@ import plugins.session
PONYMAIL_FOAL_VERSION = "0.1.0"
# The following version is generated by update_version.sh
-PONYMAIL_PLUGIN_VERSION = '86c2bf0' # Must use single quotes
+PONYMAIL_PLUGIN_VERSION = 'ddb0af0' # Must use single quotes
# Certain environments such as MinGW-w64 will not register as a TTY and uses buffered output.
[incubator-ponymail-foal] 01/02: Don't fetch fields that aren't needed
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit ddb0af055952cb3d6ba36f57caf0d07f12a9d1d0
Author: Sebb <se...@apache.org>
AuthorDate: Mon Nov 22 22:50:13 2021 +0000
Don't fetch fields that aren't needed
---
server/endpoints/stats.py | 14 +++++++++-----
server/plugins/messages.py | 12 ++++++++----
2 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py
index dbf6e4c..b25775c 100644
--- a/server/endpoints/stats.py
+++ b/server/endpoints/stats.py
@@ -59,15 +59,11 @@ async def process(
query_since = query_defuzzed.copy()
query_since['must'].append({"range" : { "epoch": { "gt": epoch}}})
results = await plugins.messages.query(
- session, query_since, query_limit=1, metadata_only=True
+ session, query_since, query_limit=1, source_fields=[''] # don't need any fields
)
if len(results) == 0:
return {"changed" : False}
- results = await plugins.messages.query(
- session, query_defuzzed, query_limit=server.config.database.max_hits
- )
-
# statsOnly: Whether to only send statistical info (for n-grams etc), and not the
# thread struct and message bodies
# Param: quick
@@ -76,6 +72,14 @@ async def process(
# i.e. omit thread_struct, top 10 participants and word-cloud
emailsOnly = 'emailsOnly' in indata
+ source_fields = None
+ if statsOnly:
+ source_fields = ['epoch']
+
+ results = await plugins.messages.query(
+ session, query_defuzzed, query_limit=server.config.database.max_hits, source_fields=source_fields
+ )
+
wordcloud = None
if server.config.ui.wordcloud and not emailsOnly and not statsOnly:
wordcloud = await plugins.messages.wordcloud(session, query_defuzzed)
diff --git a/server/plugins/messages.py b/server/plugins/messages.py
index baf6e1a..f255f18 100644
--- a/server/plugins/messages.py
+++ b/server/plugins/messages.py
@@ -318,7 +318,8 @@ async def query(
query_limit=10000,
hide_deleted=True,
metadata_only=False,
- epoch_order="desc"
+ epoch_order="desc",
+ source_fields=None
):
"""
Advanced query and grab for stats.py
@@ -334,6 +335,8 @@ async def query(
}
if metadata_only: # Only doc IDs and AAA fields.
es_query["_source"] = ["deleted", "private", "mid", "dbid", "list_raw"]
+ elif source_fields:
+ es_query["_source"] = source_fields
else:
es_query["_source"] = { "excludes": ["body"] }
async for hit in session.database.scan(
@@ -344,14 +347,15 @@ async def query(
# If email was delete/hidden and we're not doing an admin query, ignore it
if hide_deleted and doc.get("deleted", False):
continue
- doc["id"] = doc["mid"]
if plugins.aaa.can_access_email(session, doc):
+ if "mid" in doc: # might be missing when using source_fields
+ doc["id"] = doc["mid"]
# Calculate gravatars if not present in source
- if not metadata_only and "gravatar" not in doc:
+ if not metadata_only and not source_fields and "gravatar" not in doc:
doc["gravatar"] = gravatar(doc)
if not session.credentials:
doc = anonymize(doc)
- if not metadata_only:
+ if "body_short" in doc:
# The body_short field is set to SHORT_BODY_MAX_LEN+1 if the body is longer
# than SHORT_BODY_MAX_LEN, so we know if it has been truncated
if len(doc["body_short"] or "") > SHORT_BODY_MAX_LEN: