You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2021/03/29 13:10:37 UTC
[incubator-ponymail-foal] branch master updated (d595961 -> 7687879)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from d595961 tweak git ignores
new 08a678f add flag for enabling/disabling the online management api
new fb41a3f enforce bool
new 73945a0 Create mappings for audit log
new 2ed4f1a add hardcoded auditlog dbname
new 9665d0b let ES handle doc ID
new 3d629af add list id, this can help with searching audit logs
new 92048a2 add audit log to tools' ES mod
new a9607d1 add audit trail when indexing email
new 49bea18 add remote ip to session object
new 22502b9 default to not showing emails with "deleted" set to true
new aac2f6a don't show email if deleted/hidden
new 922069e get_email can return nothing..
new 718b6d7 don't thread in deleted emails
new 7b14a53 Start working on mgmt portal
new f5b7d82 linting
new d3389c9 PEP8 linting
new f59b376 PEP8 linting
new 4caefa8 Simple utility for pushing previous failures to ES
new 7687879 switch out logo with foal one
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
INSTALL.md | 14 +++++++++
server/endpoints/compose.py | 36 ++++++++++-----------
server/endpoints/email.py | 23 +++++---------
server/endpoints/mbox.py | 13 ++------
server/endpoints/mgmt.py | 68 ++++++++++++++++++++++++++++++++++++++++
server/endpoints/oauth.py | 24 ++++++--------
server/endpoints/pminfo.py | 4 +--
server/endpoints/preferences.py | 10 +++---
server/endpoints/source.py | 12 +++----
server/endpoints/stats.py | 24 +++-----------
server/endpoints/thread.py | 9 ++----
server/plugins/configuration.py | 2 ++
server/plugins/database.py | 1 +
server/plugins/mbox.py | 8 ++++-
server/plugins/session.py | 4 +++
tools/archiver.py | 15 +++++++++
tools/mappings.yaml | 18 +++++++++++
tools/plugins/elastic.py | 1 +
tools/push-failures.py | 60 +++++++++++++++++++++++++++++++++++
webui/images/logo.png | Bin 0 -> 31729 bytes
webui/index.html | 2 +-
webui/list.html | 2 +-
webui/oauth.html | 2 +-
webui/thread.html | 2 +-
24 files changed, 247 insertions(+), 107 deletions(-)
create mode 100644 server/endpoints/mgmt.py
create mode 100755 tools/push-failures.py
create mode 100644 webui/images/logo.png
[incubator-ponymail-foal] 01/19: add flag for enabling/disabling
the online management api
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 08a678fb05ebffc0764cfdaae7e0635d7284c8fb
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Mar 28 22:58:13 2021 +0200
add flag for enabling/disabling the online management api
---
server/plugins/configuration.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/server/plugins/configuration.py b/server/plugins/configuration.py
index 40f144b..eda54f3 100644
--- a/server/plugins/configuration.py
+++ b/server/plugins/configuration.py
@@ -19,6 +19,7 @@ class UIConfig:
mailhost: str
sender_domains: str
traceback: bool
+ mgmt_enabled: bool
def __init__(self, subyaml: dict):
self.wordcloud = bool(subyaml.get("wordcloud", False))
@@ -29,6 +30,7 @@ class UIConfig:
# Default to spitting out traceback to web clients
# Set to false in yaml to redirect to stderr instead.
self.traceback = subyaml.get("traceback", True)
+ self.mgmt_enabled = subyaml.get("mgmtconsole", False) # Whether to enable online mgmt component or not
class OAuthConfig:
[incubator-ponymail-foal] 05/19: let ES handle doc ID
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 9665d0bbf82b80f39f587475eacdab434b83feec
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:12:12 2021 +0200
let ES handle doc ID
---
tools/mappings.yaml | 2 --
1 file changed, 2 deletions(-)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index d927752..1adebd0 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -138,8 +138,6 @@ source:
type: binary
auditlog:
properties:
- id:
- type: keyword
date:
format: yyyy/MM/dd HH:mm:ss
store: true
[incubator-ponymail-foal] 10/19: default to not showing emails with
"deleted" set to true
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 22502b95d59acd0f44ffc8145f3df6668f052eea
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:30:49 2021 +0200
default to not showing emails with "deleted" set to true
This means we can have emails stored for provenance reasons, but not
shown to anyone but admins.
---
server/plugins/mbox.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index ef8f548..a6984d2 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -301,6 +301,7 @@ async def query(
query_defuzzed,
query_limit=10000,
shorten=False,
+ hide_deleted=True,
):
"""
Advanced query and grab for stats.py
@@ -315,6 +316,9 @@ async def query(
},
):
doc = hit["_source"]
+ # If email was delete/hidden and we're not doing an admin query, ignore it
+ if hide_deleted and doc.get("deleted", False):
+ continue
doc["id"] = doc["mid"]
if plugins.aaa.can_access_email(session, doc):
if not session.credentials:
[incubator-ponymail-foal] 11/19: don't show email if deleted/hidden
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit aac2f6a7848cdf8dcd2056358ca11b776ddff3ec
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:39:56 2021 +0200
don't show email if deleted/hidden
---
server/endpoints/email.py | 2 +-
server/endpoints/source.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/endpoints/email.py b/server/endpoints/email.py
index 97b60dd..fb534a4 100644
--- a/server/endpoints/email.py
+++ b/server/endpoints/email.py
@@ -41,7 +41,7 @@ async def process(
email = await plugins.mbox.get_email(session, messageid=indata.get("id"))
# If email was found, process the request if we are allowed to display it
- if email and isinstance(email, dict):
+ if email and isinstance(email, dict) and not email.get("deleted"):
if plugins.aaa.can_access_email(session, email):
# Are we fetching an attachment?
if not indata.get("attachment"):
diff --git a/server/endpoints/source.py b/server/endpoints/source.py
index 78bddbe..44bbcc5 100644
--- a/server/endpoints/source.py
+++ b/server/endpoints/source.py
@@ -36,7 +36,7 @@ async def process(
if email is None:
email = await plugins.mbox.get_email(session, messageid=indata.get("id"))
- if email and isinstance(email, dict):
+ if email and isinstance(email, dict) and not email.get("deleted"):
if plugins.aaa.can_access_email(session, email):
source = await plugins.mbox.get_source(session, permalink=email["mid"])
if source:
[incubator-ponymail-foal] 04/19: add hardcoded auditlog dbname
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 2ed4f1a8e0937883c3cb3e0949466f89232310c2
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:10:43 2021 +0200
add hardcoded auditlog dbname
---
server/plugins/database.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/server/plugins/database.py b/server/plugins/database.py
index 1a7b651..8255b97 100644
--- a/server/plugins/database.py
+++ b/server/plugins/database.py
@@ -35,6 +35,7 @@ class DBNames:
self.account = f"{dbprefix}-account"
self.session = f"{dbprefix}-session"
self.notification = f"{dbprefix}-notification"
+ self.auditlog = f"{dbprefix}-auditlog"
DBError = elasticsearch.ElasticsearchException
[incubator-ponymail-foal] 13/19: don't thread in deleted emails
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 718b6d74f65dbbe0aaa783bee3ac11471cce0e4c
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:43:52 2021 +0200
don't thread in deleted emails
---
server/plugins/mbox.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index e89cfe2..9e54f0a 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -143,6 +143,8 @@ async def fetch_children(session, pdoc, counter=0, pdocs=None, short=False):
emails = []
for doc in docs or []:
# Make sure email is accessible
+ if doc.get("deleted"):
+ continue
if plugins.aaa.can_access_email(session, doc):
if doc["mid"] not in pdocs:
mykids, myemails, pdocs = await fetch_children(
[incubator-ponymail-foal] 08/19: add audit trail when indexing email
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit a9607d164a4f8cea08f701f2a641872df9e33a23
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:27:20 2021 +0200
add audit trail when indexing email
---
tools/archiver.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/tools/archiver.py b/tools/archiver.py
index eab73fa..a940d2f 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -568,6 +568,21 @@ class Archiver(object): # N.B. Also used by import-mbox.py
"source": mbox_source(raw_message),
},
)
+
+ # Write to audit log
+ elastic.index(
+ index=elastic.db_auditlog,
+ body={
+ "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())),
+ "action": "index",
+ "remote": "internal",
+ "author": "archiver.py",
+ "target": ojson["mid"],
+ "lid": lid,
+ "log": f"Indexed email {ojson['message-id']} for {lid} as {ojson['mid']}",
+ }
+ )
+
# If we have a dump dir and ES failed, push to dump dir instead as a JSON object
# We'll leave it to another process to pick up the slack.
except Exception as err:
[incubator-ponymail-foal] 16/19: PEP8 linting
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit d3389c91f76c5f0cd04633b5bb49b1873fb3c22e
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 14:52:48 2021 +0200
PEP8 linting
---
server/endpoints/mgmt.py | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/server/endpoints/mgmt.py b/server/endpoints/mgmt.py
index a4d8f81..f81394a 100644
--- a/server/endpoints/mgmt.py
+++ b/server/endpoints/mgmt.py
@@ -27,29 +27,25 @@ import time
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Optional[dict]:
- action = indata.get('action')
- docs = indata.get('documents', [])
- doc = indata.get('document')
+ action = indata.get("action")
+ docs = indata.get("documents", [])
+ doc = indata.get("document")
if not docs and doc:
docs = [doc]
if not session.credentials.admin or not server.config.ui.mgmt_enabled:
return aiohttp.web.Response(headers={}, status=403, text="You need administrative access to use this feature.")
# Deleting/hiding a document?
- if action == 'delete':
+ if action == "delete":
delcount = 0
for doc in docs:
email = await plugins.mbox.get_email(session, permalink=doc)
if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email):
- email['deleted'] = True
+ email["deleted"] = True
await session.database.index(
- index=session.database.dbs.mbox,
- body=email,
- id=email['id'],
+ index=session.database.dbs.mbox, body=email, id=email["id"],
)
lid = email.get("list_raw")
await session.database.index(
@@ -62,7 +58,7 @@ async def process(
"target": doc,
"lid": lid,
"log": f"Removed email {doc} from {lid} archives",
- }
+ },
)
delcount += 1
return aiohttp.web.Response(headers={}, status=200, text=f"Removed {delcount} emails from archives.")
[incubator-ponymail-foal] 03/19: Create mappings for audit log
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 73945a0c0199b54c0c3f7891e25c5f52be457aaa
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:09:50 2021 +0200
Create mappings for audit log
The audit log will contain information about changes to stored objects.
Examples could be: indexing an email, hiding attachments, changing LID.
---
tools/mappings.yaml | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index 10984b8..d927752 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -136,3 +136,21 @@ source:
type: keyword
source:
type: binary
+auditlog:
+ properties:
+ id:
+ type: keyword
+ date:
+ format: yyyy/MM/dd HH:mm:ss
+ store: true
+ type: date
+ author:
+ type: keyword
+ remote:
+ type: keyword
+ action:
+ type: keyword
+ target:
+ type: keyword
+ log:
+ type: text
[incubator-ponymail-foal] 07/19: add audit log to tools' ES mod
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 92048a2cb6003a36bc5252708e66a490a6979909
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:23:18 2021 +0200
add audit log to tools' ES mod
---
tools/plugins/elastic.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tools/plugins/elastic.py b/tools/plugins/elastic.py
index 5b77475..f2416f3 100755
--- a/tools/plugins/elastic.py
+++ b/tools/plugins/elastic.py
@@ -59,6 +59,7 @@ class Elastic:
self.db_session = self.dbname + '-session'
self.db_notification = self.dbname + '-notification'
self.db_mailinglist = self.dbname + '-mailinglist'
+ self.db_auditlog = self.dbname + '-auditlog'
self.db_version = 0
dburl = config.get('elasticsearch', 'dburl', fallback=None)
[incubator-ponymail-foal] 12/19: get_email can return nothing..
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 922069e894e7153bf879bf50bf5b54f3a1d17c16
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:42:56 2021 +0200
get_email can return nothing..
---
server/plugins/mbox.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index a6984d2..e89cfe2 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -141,7 +141,7 @@ async def fetch_children(session, pdoc, counter=0, pdocs=None, short=False):
thread = []
emails = []
- for doc in docs:
+ for doc in docs or []:
# Make sure email is accessible
if plugins.aaa.can_access_email(session, doc):
if doc["mid"] not in pdocs:
[incubator-ponymail-foal] 06/19: add list id,
this can help with searching audit logs
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 3d629afbfcde593fb94330f887c931899d1213c1
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 10:23:01 2021 +0200
add list id, this can help with searching audit logs
---
tools/mappings.yaml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index 1adebd0..4d9eb1d 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -150,5 +150,7 @@ auditlog:
type: keyword
target:
type: keyword
+ lid:
+ type: keyword
log:
type: text
[incubator-ponymail-foal] 09/19: add remote ip to session object
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 49bea18d86f7aae6664732af570f5ea8692745b9
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:28:50 2021 +0200
add remote ip to session object
this will be used for audit logs
---
server/plugins/session.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/server/plugins/session.py b/server/plugins/session.py
index 4ccd526..b10c8f3 100644
--- a/server/plugins/session.py
+++ b/server/plugins/session.py
@@ -68,6 +68,7 @@ class SessionObject:
last_accessed: int
credentials: typing.Optional[SessionCredentials]
database: typing.Optional[plugins.database.Database]
+ remote: str
server: plugins.server.BaseServer
def __init__(self, server: plugins.server.BaseServer, **kwargs):
@@ -80,6 +81,7 @@ class SessionObject:
self.credentials = None
self.cookie = str(uuid.uuid4())
self.cid = None
+ self.remote = "??"
else:
self.last_accessed = kwargs.get("last_accessed", 0)
self.credentials = SessionCredentials(kwargs.get("credentials"))
@@ -115,6 +117,7 @@ async def get_session(
# In case the session is used twice within the same loop
session = copy.copy(x_session)
session.database = await server.dbpool.get()
+ session.remote = request.remote
# Do we need to update the timestamp in ES?
if (now - session.last_accessed) > FOAL_SAVE_SESSION_INTERVAL:
@@ -126,6 +129,7 @@ async def get_session(
# If not in local memory, start a new session object
session = SessionObject(server)
session.database = await server.dbpool.get()
+ session.remote = request.remote
# If a cookie was supplied, look for a session object in ES
if session_id and session.database:
[incubator-ponymail-foal] 18/19: Simple utility for pushing
previous failures to ES
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 4caefa86faeec65f6b5c3dc1f58b1d24c60b3809
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 15:00:38 2021 +0200
Simple utility for pushing previous failures to ES
When archiving with --dumponfail, this utility program can help push
failed documents into ES at a later stage.
---
tools/push-failures.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/tools/push-failures.py b/tools/push-failures.py
new file mode 100755
index 0000000..3c15930
--- /dev/null
+++ b/tools/push-failures.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Utility for retrying docs that we failed to index earlier.
+"""
+
+import argparse
+import json
+import os
+import plugins.elastic
+
+elastic = plugins.elastic.Elastic()
+
+parser = argparse.ArgumentParser(description="Command line options.")
+parser.add_argument(
+ "--source", dest="dumpdir", help="Path to the directory containing the JSON documents that failed to index"
+)
+
+args = parser.parse_args()
+
+dumpDir = args.dumpdir if args.dumpdir else "."
+
+print("Looking for *.json files in %s" % dumpDir)
+
+files = [f for f in os.listdir(dumpDir) if os.path.isfile(os.path.join(dumpDir, f)) and f.endswith(".json")]
+
+for f in files:
+ fpath = os.path.join(dumpDir, f)
+ print("Processing %s" % fpath)
+ with open(fpath, "r") as f:
+ ojson = json.load(f)
+ if "mbox" in ojson and "mbox_source" in ojson:
+ try:
+ mid = ojson["id"]
+ except KeyError:
+ mid = ojson["mbox"]["mid"]
+ elastic.index(index=elastic.db_mbox, id=mid, body=ojson["mbox"])
+
+ elastic.index(index=elastic.db_source, id=mid, body=ojson["mbox_source"])
+
+ if "attachments" in ojson and ojson["attachments"]:
+ for k, v in ojson["attachments"].items():
+ elastic.index(index=elastic.db_attachment, id=k, body={"source": v})
+ f.close()
+ os.unlink(fpath)
+print("All done! Pushed %u documents to ES." % len(files))
[incubator-ponymail-foal] 19/19: switch out logo with foal one
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 768787949d5ccd0f26486a8755a645ded70219dd
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 15:09:42 2021 +0200
switch out logo with foal one
---
webui/images/logo.png | Bin 0 -> 31729 bytes
webui/index.html | 2 +-
webui/list.html | 2 +-
webui/oauth.html | 2 +-
webui/thread.html | 2 +-
5 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/webui/images/logo.png b/webui/images/logo.png
new file mode 100644
index 0000000..61866c0
Binary files /dev/null and b/webui/images/logo.png differ
diff --git a/webui/index.html b/webui/index.html
index 8ee5d9b..cc57204 100644
--- a/webui/index.html
+++ b/webui/index.html
@@ -39,7 +39,7 @@ the License. -->
</head>
<body onload="prime_list_index();">
<div style="text-align: center; margin: 10px auto;">
- <img src="images/logo_large.png" style="width: 128px;"/><br/>
+ <img src="images/logo.png" style="width: 320px;"/><br/>
<h2>Mail Archives</h2>
<h4>Pick a list domain to start browsing emails:</h4>
</div>
diff --git a/webui/list.html b/webui/list.html
index a3b77a9..aa5fea0 100644
--- a/webui/list.html
+++ b/webui/list.html
@@ -45,7 +45,7 @@ the License. -->
<div class="container-fluid">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header collapse navbar-collapse">
- <a class="navbar-brand" href="./" onclick="location.href='./';"><span><img src="images/logo_large.png" style="margin-top: -10px !important;" height="30" width="32"/> <span class='hidden-xs title'>Pony Mail!</span></a>
+ <a class="navbar-brand" href="./" onclick="location.href='./';"><span><img src="images/foal.png" style="margin-top: -10px !important;" height="30"/> <span class='hidden-xs title'>Pony Mail</span></a>
</div>
diff --git a/webui/oauth.html b/webui/oauth.html
index 2d9903c..0def458 100644
--- a/webui/oauth.html
+++ b/webui/oauth.html
@@ -26,7 +26,7 @@
<div class="row">
<div id="bread" class="col-md-12" style="text-align: center">
- <img src="images/logo_large.png" width="256" height="256"/><br/>
+ <img src="images/logo.png" style="width: 320px;"/><br/>
<p>
<h3>Log in using one of the following identity providers:</h3>
<br/>
diff --git a/webui/thread.html b/webui/thread.html
index c3416b5..33df391 100644
--- a/webui/thread.html
+++ b/webui/thread.html
@@ -44,7 +44,7 @@ the License. -->
<div class="container-fluid">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header collapse navbar-collapse">
- <a class="navbar-brand" href="./"><span><img src="images/logo_large.png" style="margin-top: -10px !important;" height="30" width="32"/> <span class='hidden-xs title'>Pony Mail!</span></a>
+ <a class="navbar-brand" href="./"><span><img src="images/foal.png" style="margin-top: -10px !important;" height="30"/> <span class='hidden-xs title'>Pony Mail!</span></a>
</div>
[incubator-ponymail-foal] 02/19: enforce bool
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit fb41a3f702aa4bfa4d37ac403323638e9e9576ba
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Mar 28 22:58:46 2021 +0200
enforce bool
---
server/plugins/configuration.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/plugins/configuration.py b/server/plugins/configuration.py
index eda54f3..24f9bfb 100644
--- a/server/plugins/configuration.py
+++ b/server/plugins/configuration.py
@@ -30,7 +30,7 @@ class UIConfig:
# Default to spitting out traceback to web clients
# Set to false in yaml to redirect to stderr instead.
self.traceback = subyaml.get("traceback", True)
- self.mgmt_enabled = subyaml.get("mgmtconsole", False) # Whether to enable online mgmt component or not
+ self.mgmt_enabled = bool(subyaml.get("mgmtconsole", False)) # Whether to enable online mgmt component or not
class OAuthConfig:
[incubator-ponymail-foal] 14/19: Start working on mgmt portal
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 7b14a53925f2a6b7a3e58b163a13e54d6c3da9d9
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 11:48:33 2021 +0200
Start working on mgmt portal
This will cover simple operations like removing/renaming emails and
such.
---
INSTALL.md | 14 ++++++++++
server/endpoints/mgmt.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 84 insertions(+)
diff --git a/INSTALL.md b/INSTALL.md
index 52653b7..0f87b76 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -102,6 +102,20 @@ oauth:
- myoauthprovider.tld
~~~
+For administrative access to certain features, such as deleting/moving email via the UI,
+you can set a list of people who, via an authoritative oauth provider, will have access to
+this, as such:
+
+~~~yaml
+oauth:
+ authoritative_domains:
+ - googleapis.com
+ admins:
+ - humbedooh@gmail.com
+ - example@gmail.com
+~~~
+
+
Currently, you will also need to enable or tweak your `webui/js/config.js` file to match your
choice of OAuth providers, though that is subject to change.
diff --git a/server/endpoints/mgmt.py b/server/endpoints/mgmt.py
new file mode 100644
index 0000000..0ae4adf
--- /dev/null
+++ b/server/endpoints/mgmt.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Management endpoint for GDPR operations"""
+
+import plugins.server
+import plugins.session
+import plugins.mbox
+import plugins.defuzzer
+import typing
+import aiohttp.web
+import time
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> typing.Optional[dict]:
+ action = indata.get('action')
+ docs = indata.get('documents', [])
+ doc = indata.get('document')
+ if not docs and doc:
+ docs = [doc]
+ if not session.credentials.admin or not server.config.ui.mgmt_enabled:
+ return aiohttp.web.Response(headers={}, status=403, text="You need administrative access to use this feature.")
+
+ # Deleting/hiding a document?
+ if action == 'delete':
+ delcount = 0
+ for doc in docs:
+ email = await plugins.mbox.get_email(session, permalink=doc)
+ if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email):
+ email['deleted'] = True
+ await session.database.index(
+ index=session.database.dbs.mbox,
+ body=email,
+ id=email['id'],
+ )
+ lid = email.get("list_raw")
+ await session.database.index(
+ index=session.database.dbs.auditlog,
+ body={
+ "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())),
+ "action": "delete",
+ "remote": session.remote,
+ "author": f"{session.credentials.uid}@{session.credentials.oauth_provider}",
+ "target": doc,
+ "lid": lid,
+ "log": f"Removed email {doc} from {lid} archives",
+ }
+ )
+ delcount += 1
+ return aiohttp.web.Response(headers={}, status=200, text=f"Removed {delcount} emails from archives.")
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 15/19: linting
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit f5b7d82d68e3acbfc945f08a3fda4ac99c2d5407
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 14:52:02 2021 +0200
linting
---
server/endpoints/mgmt.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/server/endpoints/mgmt.py b/server/endpoints/mgmt.py
index 0ae4adf..a4d8f81 100644
--- a/server/endpoints/mgmt.py
+++ b/server/endpoints/mgmt.py
@@ -25,6 +25,7 @@ import typing
import aiohttp.web
import time
+
async def process(
server: plugins.server.BaseServer,
session: plugins.session.SessionObject,
@@ -66,5 +67,6 @@ async def process(
delcount += 1
return aiohttp.web.Response(headers={}, status=200, text=f"Removed {delcount} emails from archives.")
+
def register(server: plugins.server.BaseServer):
return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 17/19: PEP8 linting
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit f59b37678d7b0ee1671c5dc63138959d06a5e28d
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Mar 29 14:55:12 2021 +0200
PEP8 linting
---
server/endpoints/compose.py | 36 +++++++++++++++++-------------------
server/endpoints/email.py | 21 ++++++---------------
server/endpoints/mbox.py | 13 +++----------
server/endpoints/oauth.py | 24 +++++++++---------------
server/endpoints/pminfo.py | 4 +---
server/endpoints/preferences.py | 10 ++++------
server/endpoints/source.py | 10 +++-------
server/endpoints/stats.py | 24 +++++-------------------
server/endpoints/thread.py | 9 +++------
9 files changed, 51 insertions(+), 100 deletions(-)
diff --git a/server/endpoints/compose.py b/server/endpoints/compose.py
index 003ea9e..bb5c75a 100644
--- a/server/endpoints/compose.py
+++ b/server/endpoints/compose.py
@@ -26,9 +26,7 @@ import aiohttp.web
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Union[dict, aiohttp.web.Response]:
if not server.config.ui.mailhost:
@@ -37,15 +35,15 @@ async def process(
# Figure out outgoing MTA
mailhost = server.config.ui.mailhost
mailport = 25
- if ':' in mailhost:
- mailhost, _mailport = mailhost.split(':', 1)
+ if ":" in mailhost:
+ mailhost, _mailport = mailhost.split(":", 1)
mailport = int(_mailport)
# Figure out if recipient list is on allowed list
- to = indata.get('to', '')
- mldomain = to.strip("<>").split('@')[-1]
+ to = indata.get("to", "")
+ mldomain = to.strip("<>").split("@")[-1]
allowed_to_send = False
- for allowed_domain in server.config.ui.sender_domains.split(' '):
+ for allowed_domain in server.config.ui.sender_domains.split(" "):
if fnmatch.fnmatch(mldomain, allowed_domain):
allowed_to_send = True
break
@@ -54,22 +52,22 @@ async def process(
# If logged in and everything, prep for dispatch
if session.credentials and session.credentials.authoritative:
- subject = indata.get('subject')
- body = indata.get('body')
- irt = indata.get('in-repl-to')
- references = indata.get('references')
+ subject = indata.get("subject")
+ body = indata.get("body")
+ irt = indata.get("in-repl-to")
+ references = indata.get("references")
if to and subject and body:
msg = email.message.EmailMessage()
if irt:
- msg['in-reply-to'] = irt
+ msg["in-reply-to"] = irt
if references:
- msg['references'] = references
- msg['from'] = "%s <%s>" % (session.credentials.name, session.credentials.email)
- msg['to'] = to
- msg['subject'] = subject
- msg['X-Sender'] = "Apache Pony Mail Foal Composer v/0.1"
- msg.set_charset('utf-8')
+ msg["references"] = references
+ msg["from"] = "%s <%s>" % (session.credentials.name, session.credentials.email)
+ msg["to"] = to
+ msg["subject"] = subject
+ msg["X-Sender"] = "Apache Pony Mail Foal Composer v/0.1"
+ msg.set_charset("utf-8")
msg.set_content(body)
await aiosmtplib.send(msg, hostname=mailhost, port=mailport)
return {"okay": True, "message": "Message dispatched!"}
diff --git a/server/endpoints/email.py b/server/endpoints/email.py
index fb534a4..c80351f 100644
--- a/server/endpoints/email.py
+++ b/server/endpoints/email.py
@@ -27,10 +27,9 @@ import plugins.aaa
import base64
import typing
+
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Union[dict, aiohttp.web.Response]:
# First, assume permalink and look up the email based on that
@@ -57,26 +56,18 @@ async def process(
"Content-Length": str(entry.get("size")),
}
if "image/" not in ct and "text/" not in ct:
- headers[
- "Content-Disposition"
- ] = f"attachment; filename=\"{entry.get('filename')}\""
+ headers["Content-Disposition"] = f"attachment; filename=\"{entry.get('filename')}\""
try:
assert session.database, "Database not connected!"
attachment = await session.database.get(
index=session.database.dbs.attachment, id=indata.get("file")
)
if attachment:
- blob = base64.decodebytes(
- attachment["_source"].get("source").encode("utf-8")
- )
- return aiohttp.web.Response(
- headers=headers, status=200, body=blob
- )
+ blob = base64.decodebytes(attachment["_source"].get("source").encode("utf-8"))
+ return aiohttp.web.Response(headers=headers, status=200, body=blob)
except plugins.database.DBError:
pass # attachment not found
- return aiohttp.web.Response(
- headers={}, status=404, text="Attachment not found"
- )
+ return aiohttp.web.Response(headers={}, status=404, text="Attachment not found")
return aiohttp.web.Response(headers={}, status=404, text="Email not found")
diff --git a/server/endpoints/mbox.py b/server/endpoints/mbox.py
index 14b811e..aff4ee8 100644
--- a/server/endpoints/mbox.py
+++ b/server/endpoints/mbox.py
@@ -26,15 +26,11 @@ import aiohttp.web
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Union[dict, aiohttp.web.Response]:
query_defuzzed = plugins.defuzzer.defuzz(indata)
- results = await plugins.mbox.query(
- session, query_defuzzed, query_limit=server.config.database.max_hits,
- )
+ results = await plugins.mbox.query(session, query_defuzzed, query_limit=server.config.database.max_hits,)
sources = []
for email in results:
@@ -58,10 +54,7 @@ async def process(
# Return mbox archive with filename
return aiohttp.web.Response(
- headers={
- "Content-Type": "application/mbox",
- "Content-Disposition": f"attachment; filename={dlfile}",
- },
+ headers={"Content-Type": "application/mbox", "Content-Disposition": f"attachment; filename={dlfile}",},
status=200,
text="\n\n".join(sources),
)
diff --git a/server/endpoints/oauth.py b/server/endpoints/oauth.py
index 8ab02f6..414c682 100644
--- a/server/endpoints/oauth.py
+++ b/server/endpoints/oauth.py
@@ -28,27 +28,24 @@ import hashlib
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Union[dict, aiohttp.web.Response]:
state = indata.get("state")
code = indata.get("code")
- id_token = indata.get('id_token')
+ id_token = indata.get("id_token")
oauth_token = indata.get("oauth_token")
rv: typing.Optional[dict] = None
# Google OAuth - currently fetches email address only
- if indata.get('key', '') == 'google' and id_token:
+ if indata.get("key", "") == "google" and id_token:
rv = await plugins.oauthGoogle.process(indata, session, server)
# GitHub OAuth - Fetches name and email
- if indata.get('key', '') == 'github' and code:
+ if indata.get("key", "") == "github" and code:
rv = await plugins.oauthGithub.process(indata, session, server)
-
# Generic OAuth handler, only one we support for now. Works with ASF OAuth.
elif state and code and oauth_token:
rv = await plugins.oauthGeneric.process(indata, session, server)
@@ -60,12 +57,10 @@ async def process(
uid = rv.get("email")
if uid:
cid = hashlib.shake_128(
- ("%s-%s" % (rv.get("oauth_domain", "generic"), uid)).encode(
- "ascii", "ignore"
- )
+ ("%s-%s" % (rv.get("oauth_domain", "generic"), uid)).encode("ascii", "ignore")
).hexdigest(16)
authoritative = rv.get("oauth_domain", "generic") in server.config.oauth.authoritative_domains
- admin = authoritative and rv.get('email') in server.config.oauth.admins
+ admin = authoritative and rv.get("email") in server.config.oauth.admins
cookie = await plugins.session.set_session(
server,
cid,
@@ -77,16 +72,15 @@ async def process(
authoritative=authoritative,
oauth_provider=rv.get("oauth_domain", "generic"),
oauth_data=rv,
- admin=admin
+ admin=admin,
)
# This could be improved upon, instead of a raw response return value
return aiohttp.web.Response(
- headers={"set-cookie": cookie, "content-type": "application/json"},
- status=200,
- text='{"okay": true}',
+ headers={"set-cookie": cookie, "content-type": "application/json"}, status=200, text='{"okay": true}',
)
return {"okay": False, "message": "Could not process OAuth login!"}
+
def register(server: plugins.server.BaseServer):
return plugins.server.Endpoint(process)
diff --git a/server/endpoints/pminfo.py b/server/endpoints/pminfo.py
index 21bf996..98e6f48 100644
--- a/server/endpoints/pminfo.py
+++ b/server/endpoints/pminfo.py
@@ -20,9 +20,7 @@
import plugins.server
-async def process(
- server: plugins.server.BaseServer, session: dict, indata: dict
-) -> dict:
+async def process(server: plugins.server.BaseServer, session: dict, indata: dict) -> dict:
return server.data.activity
diff --git a/server/endpoints/preferences.py b/server/endpoints/preferences.py
index 7a473aa..1cb2821 100644
--- a/server/endpoints/preferences.py
+++ b/server/endpoints/preferences.py
@@ -23,9 +23,7 @@ import plugins.session
""" This is incomplete, but will work for anonymous tests. """
-async def process(
- server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict
-) -> dict:
+async def process(server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict) -> dict:
prefs: dict = {"login": {}}
lists: dict = {}
for ml, entry in server.data.lists.items():
@@ -40,7 +38,7 @@ async def process(
lists[ldomain][lname] = entry["count"]
prefs["lists"] = lists
if session and session.credentials:
- prefs['login'] = {
+ prefs["login"] = {
"credentials": {
"uid": session.credentials.uid,
"email": session.credentials.email,
@@ -48,10 +46,10 @@ async def process(
}
}
if session.credentials.admin is True:
- prefs['login']['credentials']['admin'] = True
+ prefs["login"]["credentials"]["admin"] = True
# Logging out??
- if indata.get('logout'):
+ if indata.get("logout"):
# Remove session from ElasticSearch
await plugins.session.remove_session(session)
diff --git a/server/endpoints/source.py b/server/endpoints/source.py
index 44bbcc5..60fa085 100644
--- a/server/endpoints/source.py
+++ b/server/endpoints/source.py
@@ -25,9 +25,7 @@ import plugins.aaa
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> aiohttp.web.Response:
# First, assume permalink and look up the email based on that
email = await plugins.mbox.get_email(session, permalink=indata.get("id"))
@@ -35,15 +33,13 @@ async def process(
# If not found via permalink, it might be message-id instead, so try that
if email is None:
email = await plugins.mbox.get_email(session, messageid=indata.get("id"))
-
+
if email and isinstance(email, dict) and not email.get("deleted"):
if plugins.aaa.can_access_email(session, email):
source = await plugins.mbox.get_source(session, permalink=email["mid"])
if source:
return aiohttp.web.Response(
- headers={"Content-Type": "text/plain"},
- status=200,
- text=source["_source"]["source"],
+ headers={"Content-Type": "text/plain"}, status=200, text=source["_source"]["source"],
)
return aiohttp.web.Response(headers={}, status=404, text="Email not found")
diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py
index ed328fd..40746b7 100644
--- a/server/endpoints/stats.py
+++ b/server/endpoints/stats.py
@@ -30,19 +30,12 @@ import typing
PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
-async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
-) -> dict:
+async def process(server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,) -> dict:
query_defuzzed = plugins.defuzzer.defuzz(indata)
query_defuzzed_nodate = plugins.defuzzer.defuzz(indata, nodate=True)
results = await plugins.mbox.query(
- session,
- query_defuzzed,
- query_limit=server.config.database.max_hits,
- shorten=True,
+ session, query_defuzzed, query_limit=server.config.database.max_hits, shorten=True,
)
for msg in results:
@@ -58,20 +51,13 @@ async def process(
xlist = indata.get("list", "*")
xdomain = indata.get("domain", "*")
- all_authors = sorted(
- [[author, count] for author, count in authors.items()], key=lambda x: x[1]
- )
+ all_authors = sorted([[author, count] for author, count in authors.items()], key=lambda x: x[1])
top10_authors = []
for x in [x for x in reversed([x for x in all_authors])][:10]:
author, count = x
name, address = email.utils.parseaddr(author)
top10_authors.append(
- {
- "email": address,
- "name": name,
- "count": count,
- "gravatar": plugins.mbox.gravatar(author),
- }
+ {"email": address, "name": name, "count": count, "gravatar": plugins.mbox.gravatar(author),}
)
# Trim email data so as to reduce download sizes
@@ -84,7 +70,7 @@ async def process(
"hits": len(results),
"numparts": len(authors),
"no_threads": len(tstruct),
- "emails": list(sorted(results, key=lambda x: x['epoch'])),
+ "emails": list(sorted(results, key=lambda x: x["epoch"])),
"cloud": wordcloud,
"participants": top10_authors,
"thread_struct": tstruct,
diff --git a/server/endpoints/thread.py b/server/endpoints/thread.py
index af3cc1a..b583bbe 100644
--- a/server/endpoints/thread.py
+++ b/server/endpoints/thread.py
@@ -23,18 +23,15 @@ import plugins.mbox
import plugins.defuzzer
import typing
+
async def process(
- server: plugins.server.BaseServer,
- session: plugins.session.SessionObject,
- indata: dict,
+ server: plugins.server.BaseServer, session: plugins.session.SessionObject, indata: dict,
) -> typing.Optional[dict]:
email = await plugins.mbox.get_email(session, permalink=indata.get("id"))
if not email:
email = await plugins.mbox.get_email(session, messageid=indata.get("id"))
if email and isinstance(email, dict):
- thread, emails, pdocs = await plugins.mbox.fetch_children(
- session, email, short=True
- )
+ thread, emails, pdocs = await plugins.mbox.fetch_children(session, email, short=True)
else:
return None