You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by se...@apache.org on 2021/12/13 17:38:32 UTC
[incubator-ponymail-foal] 01/02: Rename to agree with elastic/Elastic class
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 475e3a7adfb6f1812111cef42452b8c60719e644
Author: Sebb <se...@apache.org>
AuthorDate: Mon Dec 13 17:37:14 2021 +0000
Rename to agree with elastic/Elastic class
Also, the prefix makes it easier to distinguish db index references
---
server/endpoints/email.py | 2 +-
server/endpoints/gravatar.py | 2 +-
server/endpoints/mgmt.py | 18 +++++++++---------
server/plugins/auditlog.py | 6 +++---
server/plugins/background.py | 20 ++++++++++----------
server/plugins/database.py | 24 ++++++++++++------------
server/plugins/messages.py | 10 +++++-----
server/plugins/session.py | 12 ++++++------
8 files changed, 47 insertions(+), 47 deletions(-)
diff --git a/server/endpoints/email.py b/server/endpoints/email.py
index 1ff6fdd..22f2343 100644
--- a/server/endpoints/email.py
+++ b/server/endpoints/email.py
@@ -67,7 +67,7 @@ async def process(
try:
assert session.database, "Database not connected!"
attachment = await session.database.get(
- index=session.database.dbs.attachment, id=indata.get("file")
+ index=session.database.dbs.db_attachment, id=indata.get("file")
)
if attachment:
blob = base64.decodebytes(attachment["_source"].get("source").encode("utf-8"))
diff --git a/server/endpoints/gravatar.py b/server/endpoints/gravatar.py
index 6602596..c0b1cf9 100644
--- a/server/endpoints/gravatar.py
+++ b/server/endpoints/gravatar.py
@@ -44,7 +44,7 @@ async def fetch_gravatar(gid):
async def gravatar_exists_in_db(session, gid):
res = await session.database.search(
- index=session.database.dbs.mbox,
+ index=session.database.dbs.db_mbox,
size=1,
body={"query": {"bool": {"must": [{"term": {"gravatar": gid}}]}}},
)
diff --git a/server/endpoints/mgmt.py b/server/endpoints/mgmt.py
index 6ac0d67..28335ea 100644
--- a/server/endpoints/mgmt.py
+++ b/server/endpoints/mgmt.py
@@ -58,14 +58,14 @@ async def process(
email["deleted"] = True
if server.config.ui.fully_delete and email["id"] and email["dbid"]: # Full on GDPR blast?
await session.database.delete(
- index=session.database.dbs.mbox, id=email["id"],
+ index=session.database.dbs.db_mbox, id=email["id"],
)
await session.database.delete(
- index=session.database.dbs.source, id=email["dbid"],
+ index=session.database.dbs.db_source, id=email["dbid"],
)
else: # Standard behavior: hide the email from everyone.
await session.database.update(
- index=session.database.dbs.mbox, body={"doc": email}, id=email["id"],
+ index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"],
)
lid = email.get("list_raw", "??")
await plugins.auditlog.add_entry(session, action="delete", target=doc, lid=lid, log=f"Removed email {doc} from {lid} archives")
@@ -80,7 +80,7 @@ async def process(
if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email):
email["deleted"] = True
await session.database.update(
- index=session.database.dbs.mbox, body={"doc": email}, id=email["id"],
+ index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"],
)
lid = email.get("list_raw", "??")
await plugins.auditlog.add_entry(session, action="hide", target=doc, lid=lid, log=f"Hid email {doc} from {lid} archives")
@@ -95,7 +95,7 @@ async def process(
if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email):
email["deleted"] = False
await session.database.update(
- index=session.database.dbs.mbox, body={"doc": email}, id=email["id"],
+ index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"],
)
lid = email.get("list_raw", "??")
await plugins.auditlog.add_entry(session, action="unhide", target=doc, lid=lid, log=f"Unhid email {doc} from {lid} archives")
@@ -110,14 +110,14 @@ async def process(
try:
assert session.database, "Database not connected!"
attachment = await session.database.get(
- index=session.database.dbs.attachment, id=doc
+ index=session.database.dbs.db_attachment, id=doc
)
except plugins.database.DBError:
pass # attachment not found
if attachment and isinstance(attachment, dict):
await session.database.delete(
- index=session.database.dbs.attachment, id=attachment["_id"],
+ index=session.database.dbs.db_attachment, id=attachment["_id"],
)
lid = "<system>"
await plugins.auditlog.add_entry(session, action="delatt", target=doc, lid=lid, log=f"Removed attachment {doc} from the archives")
@@ -164,7 +164,7 @@ async def process(
# Save edited email
await session.database.update(
- index=session.database.dbs.mbox, body={"doc": email}, id=email["id"],
+ index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"],
)
# Fetch source, mark as deleted (modified) and save IF anything but just privacy changed
@@ -176,7 +176,7 @@ async def process(
source = source["_source"]
source["deleted"] = True
await session.database.update(
- index=session.database.dbs.source, body={"doc": source}, id=docid,
+ index=session.database.dbs.db_source, body={"doc": source}, id=docid,
)
await plugins.auditlog.add_entry(session, action="edit", target=doc, lid=lid,
diff --git a/server/plugins/auditlog.py b/server/plugins/auditlog.py
index ad94d16..955ba82 100644
--- a/server/plugins/auditlog.py
+++ b/server/plugins/auditlog.py
@@ -50,11 +50,11 @@ async def view(
assert session.database, "No database connection could be found!"
if not filter:
res = await session.database.search(
- index=session.database.dbs.auditlog, size=num_entries, from_=page * num_entries, sort="date:desc",
+ index=session.database.dbs.db_auditlog, size=num_entries, from_=page * num_entries, sort="date:desc",
)
else:
res = await session.database.search(
- index=session.database.dbs.auditlog, size=num_entries, from_=page * num_entries, sort="date:desc",
+ index=session.database.dbs.db_auditlog, size=num_entries, from_=page * num_entries, sort="date:desc",
body={
"query": {"bool": {"must": [{"terms": {"action": filter}}]}}
},
@@ -79,7 +79,7 @@ async def add_entry(session: plugins.session.SessionObject, action: str, target:
assert session.credentials, "No session credentials could be found!"
assert session.database, "Session not connected to database!"
await session.database.index(
- index=session.database.dbs.auditlog,
+ index=session.database.dbs.db_auditlog,
body={
"date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())),
"action": action,
diff --git a/server/plugins/background.py b/server/plugins/background.py
index 34296a3..72c626d 100644
--- a/server/plugins/background.py
+++ b/server/plugins/background.py
@@ -63,13 +63,13 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict:
# Fetch aggregations of all private emails
# Do this first, so mixed lists are not marked private
- s = Search(using=db.client, index=db.dbs.mbox).filter(
+ s = Search(using=db.client, index=db.dbs.db_mbox).filter(
"term", private=True
)
s.aggs.bucket("per_list", "terms", field="list_raw", size=limit)
res = await db.search(
- index=db.dbs.mbox, body=s.to_dict(), size=0
+ index=db.dbs.db_mbox, body=s.to_dict(), size=0
)
for ml in res["aggregations"]["per_list"]["buckets"]:
@@ -80,13 +80,13 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict:
}
# Fetch aggregations of all public emails
- s = Search(using=db.client, index=db.dbs.mbox).filter(
+ s = Search(using=db.client, index=db.dbs.db_mbox).filter(
"term", private=False
)
s.aggs.bucket("per_list", "terms", field="list_raw", size=limit)
res = await db.search(
- index=db.dbs.mbox, body=s.to_dict(), size=0
+ index=db.dbs.db_mbox, body=s.to_dict(), size=0
)
for ml in res["aggregations"]["per_list"]["buckets"]:
@@ -97,12 +97,12 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict:
}
# Get 90 day activity, if any
- s = Search(using=db.client, index=db.dbs.mbox)
+ s = Search(using=db.client, index=db.dbs.db_mbox)
s = s.filter('range', date = {'gte': ACTIVITY_TIMESPAN})
s.aggs.bucket("per_list", "terms", field="list_raw", size=limit)
res = await db.search(
- index=db.dbs.mbox, body=s.to_dict(), size=0
+ index=db.dbs.db_mbox, body=s.to_dict(), size=0
)
for ml in res["aggregations"]["per_list"]["buckets"]:
@@ -125,7 +125,7 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict:
# Fetch aggregations of all public emails
s = (
- Search(using=db, index=db.dbs.mbox)
+ Search(using=db, index=db.dbs.db_mbox)
.query("match", private=False)
.filter("range", date={"lt": "now+1d", "gt": "now-14d"})
)
@@ -137,7 +137,7 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict:
)
res = await db.search(
- index=db.dbs.mbox, body=s.to_dict(), size=0
+ index=db.dbs.db_mbox, body=s.to_dict(), size=0
)
no_emails = res["hits"]["total"]["value"]
@@ -153,12 +153,12 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict:
thread_count = 0
s = (
- Search(using=db.client, index=db.dbs.mbox)
+ Search(using=db.client, index=db.dbs.db_mbox)
.query("match", private=False)
.filter("range", date={"lt": "now+1d", "gt": "now-14d"})
)
async for docs in db.scan(
- index=db.dbs.mbox,
+ index=db.dbs.db_mbox,
query=s.to_dict(),
_source_includes=[
"message-id",
diff --git a/server/plugins/database.py b/server/plugins/database.py
index cf1f436..eccb18b 100644
--- a/server/plugins/database.py
+++ b/server/plugins/database.py
@@ -33,13 +33,13 @@ class Timeout (elasticsearch.exceptions.ConnectionTimeout):
class DBNames:
def __init__(self, dbprefix):
- self.mbox = f"{dbprefix}-mbox"
- self.source = f"{dbprefix}-source"
- self.attachment = f"{dbprefix}-attachment"
- self.account = f"{dbprefix}-account"
- self.session = f"{dbprefix}-session"
- self.notification = f"{dbprefix}-notification"
- self.auditlog = f"{dbprefix}-auditlog"
+ self.db_mbox = f"{dbprefix}-mbox"
+ self.db_source = f"{dbprefix}-source"
+ self.db_attachment = f"{dbprefix}-attachment"
+ self.db_account = f"{dbprefix}-account"
+ self.db_session = f"{dbprefix}-session"
+ self.db_notification = f"{dbprefix}-notification"
+ self.db_auditlog = f"{dbprefix}-auditlog"
DBError = elasticsearch.ElasticsearchException
@@ -71,7 +71,7 @@ class Database:
async def search(self, index="", **kwargs):
if not index:
- index = self.dbs.mbox
+ index = self.dbs.db_mbox
try:
res = await self.client.search(index=index, **kwargs)
return res
@@ -80,19 +80,19 @@ class Database:
async def get(self, index="", **kwargs):
if not index:
- index = self.dbs.mbox
+ index = self.dbs.db_mbox
res = await self.client.get(index=index, **kwargs)
return res
async def delete(self, index="", **kwargs):
if not index:
- index = self.dbs.session
+ index = self.dbs.db_session
res = await self.client.delete(index=index, **kwargs)
return res
async def index(self, index="", **kwargs):
if not index:
- index = self.dbs.session
+ index = self.dbs.db_session
res = await self.client.index(index=index, **kwargs)
return res
@@ -108,7 +108,7 @@ class Database:
async def update(self, index="", **kwargs):
if not index:
- index = self.dbs.session
+ index = self.dbs.db_session
res = await self.client.update(index=index, **kwargs)
return res
diff --git a/server/plugins/messages.py b/server/plugins/messages.py
index 9c63458..42cb948 100644
--- a/server/plugins/messages.py
+++ b/server/plugins/messages.py
@@ -216,9 +216,9 @@ async def get_email(
source=False,
):
assert session.database, DATABASE_NOT_CONNECTED
- doctype = session.database.dbs.mbox
+ doctype = session.database.dbs.db_mbox
if source:
- doctype = session.database.dbs.source
+ doctype = session.database.dbs.db_source
# Older indexes may need a match instead of a strict terms agg in order to find
# emails in DBs that may have been incorrectly analyzed.
aggtype = "match"
@@ -287,7 +287,7 @@ async def get_email(
async def get_source(session: plugins.session.SessionObject, permalink: str = None, raw=False):
assert session.database, DATABASE_NOT_CONNECTED
- doctype = session.database.dbs.source
+ doctype = session.database.dbs.db_source
try:
doc = await session.database.get(index=doctype, id=permalink)
except plugins.database.DBError:
@@ -457,7 +457,7 @@ async def get_activity_span(session, query_defuzzed):
fuzz_private_only = dict(query_defuzzed)
fuzz_private_only["filter"] = [{"term": {"private": True}}]
res = await session.database.search(
- index=session.database.dbs.mbox,
+ index=session.database.dbs.db_mbox,
size=0,
body={
"query": {"bool": fuzz_private_only},
@@ -486,7 +486,7 @@ async def get_activity_span(session, query_defuzzed):
# Get oldest and youngest doc in single scan, as well as a monthly histogram
res = await session.database.search(
- index=session.database.dbs.mbox,
+ index=session.database.dbs.db_mbox,
size=0,
body={"query": {"bool": query_defuzzed},
"aggs": {
diff --git a/server/plugins/session.py b/server/plugins/session.py
index 4b2cbc2..ed44730 100644
--- a/server/plugins/session.py
+++ b/server/plugins/session.py
@@ -143,14 +143,14 @@ async def get_session(
if session_id and session.database:
try:
session_doc = await session.database.get(
- session.database.dbs.session, id=session_id
+ session.database.dbs.db_session, id=session_id
)
last_update = session_doc["_source"]["updated"]
session.cookie = session_id
# Check that this cookie ain't too old. If it is, delete it and return bare-bones session object
if (now - last_update) > FOAL_MAX_SESSION_AGE:
session.database.delete(
- index=session.database.dbs.session, id=session_id
+ index=session.database.dbs.db_session, id=session_id
)
return session
@@ -158,7 +158,7 @@ async def get_session(
cid = session_doc["_source"]["cid"]
if cid:
account_doc = await session.database.get(
- session.database.dbs.account, id=cid
+ session.database.dbs.db_account, id=cid
)
creds = account_doc["_source"]["credentials"]
internal = account_doc["_source"]["internal"]
@@ -212,7 +212,7 @@ async def save_session(session: SessionObject):
"""Save a session object in the ES database"""
assert session.database, DATABASE_NOT_CONNECTED
await session.database.index(
- index=session.database.dbs.session,
+ index=session.database.dbs.db_session,
id=session.cookie,
body={
"cookie": session.cookie,
@@ -225,7 +225,7 @@ async def save_session(session: SessionObject):
async def remove_session(session: SessionObject):
"""Remove a session object in the ES database"""
assert session.database, DATABASE_NOT_CONNECTED
- await session.database.delete(index=session.database.dbs.session, id=session.cookie)
+ await session.database.delete(index=session.database.dbs.db_session, id=session.cookie)
async def save_credentials(session: SessionObject):
@@ -233,7 +233,7 @@ async def save_credentials(session: SessionObject):
assert session.database, DATABASE_NOT_CONNECTED
assert session.credentials, "Session object without credentials, cannot save!"
await session.database.index(
- index=session.database.dbs.account,
+ index=session.database.dbs.db_account,
id=session.cid,
body={
"cid": session.cid,