You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/09/06 17:34:45 UTC
[incubator-ponymail-foal] branch master updated (f57702e -> d5bc8d0)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from f57702e This var is unused outside of this scope.
new 0f1210f add backend ui config
new 99a4ff5 only check this if --generator is set
new bcd7e32 Add minimal server package for UI backend
new 8a3e865 add defuzzer plugin
new a639cb0 add mbox utulity plugin
new 74d6bd0 add pminfo endpoint
new 775998e Add stats endpoint
new d1dcb5d Add single email endpoint
new 6a71ff0 Add source endpoint
new 7e45a95 Add preferences endpoint
new a724b9b needs another newline
new b3586ba Add thread endpoint
new 8150129 Add server requirements.txt
new eb798e4 Add a simple README for now, with run instructions for httpd
new d5bc8d0 permalinks with an S
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/README.md | 37 +++
server/endpoints/email.py | 79 +++++
server/endpoints/pminfo.py | 30 ++
server/endpoints/preferences.py | 46 +++
server/endpoints/source.py | 47 +++
server/endpoints/stats.py | 90 ++++++
server/endpoints/thread.py | 50 ++++
server/main.py | 175 +++++++++++
server/plugins/__init__.py | 1 +
server/plugins/aaa.py | 29 ++
server/plugins/background.py | 204 +++++++++++++
server/plugins/configuration.py | 57 ++++
server/plugins/database.py | 73 +++++
server/plugins/defuzzer.py | 175 +++++++++++
server/plugins/formdata.py | 65 +++++
server/plugins/mbox.py | 432 ++++++++++++++++++++++++++++
server/plugins/offloader.py | 43 +++
server/plugins/server.py | 25 ++
server/plugins/session.py | 111 +++++++
server/ponymail.yaml.example | 15 +
requirements.txt => server/requirements.txt | 4 +-
tools/setup.py | 22 +-
22 files changed, 1808 insertions(+), 2 deletions(-)
create mode 100644 server/README.md
create mode 100644 server/endpoints/email.py
create mode 100644 server/endpoints/pminfo.py
create mode 100644 server/endpoints/preferences.py
create mode 100644 server/endpoints/source.py
create mode 100644 server/endpoints/stats.py
create mode 100644 server/endpoints/thread.py
create mode 100644 server/main.py
create mode 100644 server/plugins/__init__.py
create mode 100644 server/plugins/aaa.py
create mode 100644 server/plugins/background.py
create mode 100644 server/plugins/configuration.py
create mode 100644 server/plugins/database.py
create mode 100644 server/plugins/defuzzer.py
create mode 100644 server/plugins/formdata.py
create mode 100644 server/plugins/mbox.py
create mode 100644 server/plugins/offloader.py
create mode 100644 server/plugins/server.py
create mode 100644 server/plugins/session.py
create mode 100644 server/ponymail.yaml.example
copy requirements.txt => server/requirements.txt (73%)
[incubator-ponymail-foal] 08/15: Add single email endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit d1dcb5d04679e7c8063cf1061021534c55e53c7d
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:20:42 2020 +0200
Add single email endpoint
---
server/endpoints/email.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/server/endpoints/email.py b/server/endpoints/email.py
new file mode 100644
index 0000000..c22bca1
--- /dev/null
+++ b/server/endpoints/email.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple endpoint that returns an email or an attachment from one"""
+""" THIS ONLY DEALS WITH PUBLIC EMAILS FOR NOW - AAA IS BEING WORKED ON"""
+
+import plugins.server
+import plugins.session
+import plugins.mbox
+import aiohttp.web
+import plugins.aaa
+import base64
+
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> dict:
+
+ email = await plugins.mbox.get_email(session, id=indata.get("id"))
+ if email and isinstance(email, dict):
+ if plugins.aaa.can_access_email(session, email):
+ # Are we fetching an attachment?
+ if not indata.get("attachment"):
+ email["gravatar"] = plugins.mbox.gravatar(email)
+ return email
+ else:
+ fid = indata.get("file")
+ for entry in email.get("attachments", []):
+ if entry.get("hash") == fid:
+ ct = entry.get("content_type") or "application/binary"
+ headers = {
+ "Content-Type": ct,
+ "Content-Length": str(entry.get("size")),
+ }
+ if "image/" not in ct and "text/" not in ct:
+ headers[
+ "Content-Disposition"
+ ] = f"attachment; filename=\"{entry.get('filename')}\""
+ try:
+ attachment = await session.database.get(
+ itype="attachment", id=indata.get("file")
+ )
+ if attachment:
+ blob = base64.decodebytes(
+ attachment["_source"].get("source").encode("utf-8")
+ )
+ return aiohttp.web.Response(
+ headers=headers, status=200, body=blob
+ )
+ except Exception as e:
+ print(e)
+ return aiohttp.web.Response(
+ headers={}, status=404, text="Attachment not found"
+ )
+
+ else:
+ return aiohttp.web.Response(headers={}, status=404, text="Email not found")
+ else:
+ return aiohttp.web.Response(headers={}, status=404, text="Email not found")
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 04/15: add defuzzer plugin
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 8a3e86518ac7561171ec7f4f2a62bcaee109b319
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:14:30 2020 +0200
add defuzzer plugin
---
server/plugins/defuzzer.py | 175 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 175 insertions(+)
diff --git a/server/plugins/defuzzer.py b/server/plugins/defuzzer.py
new file mode 100644
index 0000000..8d3e66c
--- /dev/null
+++ b/server/plugins/defuzzer.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import calendar
+import re
+import shlex
+import typing
+
+"""
+This is the query de-fuzzer library for Foal.
+It turns a URL search query into an ES query
+
+"""
+
+
+def defuzz(formdata: dict, nodate: bool = False) -> dict:
+ # Default to 30 day date range
+ daterange = {"gt": "now-30d", "lt": "now+1d"}
+
+ # Custom date range?
+ # If a month is the only thing, fake start and end
+ if "date" in formdata and "e" not in formdata:
+ formdata["s"] = formdata["date"]
+ formdata["e"] = formdata["date"]
+ # classic start and end month params
+ if "s" in formdata and "e" in formdata:
+ syear, smonth = formdata["s"].split("-")
+ eyear, emonth = formdata["e"].split("-")
+ estart, eend = calendar.monthrange(int(eyear), int(emonth))
+ daterange = {
+ "gt": "%04u/%02u/01 00:00:00" % (int(syear), int(smonth)),
+ "lt": "%04u/%02u/%02u 23:59:59" % (int(eyear), int(emonth), eend),
+ }
+ # Advanced date formatting
+ elif "d" in formdata:
+ # The more/less than N days/weeks/months/years ago
+ m = re.match(r"^([a-z]+)=([0-9Mwyd]+)$", formdata["d"])
+ if m:
+ t = m.group(1)
+ r = m.group(2)
+ if t == "lte" and r:
+ daterange = {"gt": "now-%s" % r}
+ elif t == "gte" and r:
+ daterange = {"lt": "now-%s" % r}
+ # simple one month listing
+ m = re.match(r"^(\d\d\d\d-\d+)$", formdata["d"])
+ if m:
+ xdate = m.group(1)
+ dyear, dmonth = xdate.split("-", 1)
+ daterange = {
+ "gte": "%04u-%02u-01||/M" % (int(dyear), int(dmonth)),
+ "lte": "%04u-%02u-01||/M" % (int(dyear), int(dmonth)),
+ "format": "yyyy-MM-dd",
+ }
+
+ # dfr and dto defining a time span
+ m = re.match(r"^dfr=(\d\d\d\d-\d+-\d+)\|dto=(\d\d\d\d-\d+-\d+)$", formdata["d"])
+ if m:
+ dfr = m.group(1)
+ dto = m.group(2)
+ syear, smonth, sday = dfr.split("-")
+ eyear, emonth, eday = dto.split("-")
+ daterange = {
+ "gt": "%04u/%02u/%02u 00:00:00" % (int(syear), int(smonth), int(sday)),
+ "lt": "%04u/%02u/%02u 23:59:59" % (int(eyear), int(emonth), int(eday)),
+ }
+
+ # List parameter(s)
+ if "domain" in formdata:
+ fqdn = formdata["domain"]
+ listname = formdata.get("list", "*")
+ elif "list" in formdata:
+ listname, fqdn = formdata["list"].split("@", 1)
+ else: # No domain or list at all? BORK!
+ listname = "*"
+ fqdn = "*"
+ list_raw = "<%s.%s>" % (listname, fqdn)
+
+ # Default is to look in a specific list
+ query_list_hash: typing.Dict = {"term": {"list_raw": list_raw}}
+
+ # *@fqdn match?
+ if listname == "*" and fqdn != "*":
+ query_list_hash = {"wildcard": {"list_raw": {"value": "*.%s>" % fqdn}}}
+
+ # listname@* match?
+ if listname != "*" and fqdn == "*":
+ query_list_hash = {"wildcard": {"list_raw": "<%s.*>" % listname}}
+
+ # *@* ??
+ if listname == "*" and fqdn == "*":
+ query_list_hash = {"wildcard": {"list_raw": "*"}}
+
+ must = [query_list_hash]
+ must_not = []
+
+ # Append date range if not excluded
+ if not nodate:
+ must.append({"range": {"date": daterange}})
+
+ # Query string search:
+ # - foo bar baz: find emails with these words
+ # - orange -apples: fond email with oranges but not apples
+ # - "this sentence": find emails with this exact string
+ if "q" in formdata:
+ qs = formdata["q"].replace(":", "")
+ bits = shlex.split(qs)
+
+ should = []
+ shouldnot = []
+
+ for bit in bits:
+ force_positive = False
+ # Translate -- into a positive '-', so you can find "-1" etc
+ if bit[0:1] == "--":
+ force_positive = True
+ bit = bit[1:]
+ # Negatives
+ if bit[0] == "-" and not force_positive:
+ # Quoted?
+ if bit.find(" ") != -1:
+ bit = '"' + bit + '"'
+ shouldnot.append(bit[1:])
+ # Positives
+ else:
+ # Quoted?
+ if bit.find(" ") != -1:
+ bit = '"' + bit + '"'
+ should.append(bit)
+
+ if len(should) > 0:
+ must.append(
+ {
+ "multi_match": {
+ "fields": ["subject", "from", "body"],
+ "query": " AND ".join(should),
+ }
+ }
+ )
+ if len(shouldnot) > 0:
+ must_not.append(
+ {
+ "multi_match": {
+ "fields": ["subject", "from", "body"],
+ "query": " AND ".join(shouldnot),
+ }
+ }
+ )
+
+ # Header parameters
+ for header in ["from", "subject", "body", "to"]:
+ hname = "header_%s" % header
+ if hname in formdata:
+ hvalue = formdata[hname] # .replace('"', "")
+ must.append({"match": {header: {"query": hvalue}}})
+
+ thebool = {"must": must}
+
+ if len(must_not) > 0:
+ thebool["must_not"] = must_not
+ return thebool
[incubator-ponymail-foal] 12/15: Add thread endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit b3586ba8d1b2c42810ba8003e5d32951b7a5fcea
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:23:24 2020 +0200
Add thread endpoint
---
server/endpoints/thread.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/server/endpoints/thread.py b/server/endpoints/thread.py
new file mode 100644
index 0000000..7e14c64
--- /dev/null
+++ b/server/endpoints/thread.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple endpoint that returns the server's gathered activity data"""
+
+import plugins.server
+import plugins.session
+import plugins.mbox
+import plugins.defuzzer
+
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> dict:
+ email = await plugins.mbox.get_email(session, id=indata.get("id"))
+ if not email:
+ email = await plugins.mbox.get_email(session, messageid=indata.get("id"))
+ if email and isinstance(email, dict):
+ thread, emails, pdocs = await plugins.mbox.fetch_children(
+ session, email, short=True
+ )
+ else:
+ return None
+
+ email["children"] = thread
+ emails.append(email)
+ return {
+ "thread": email,
+ "emails": emails,
+ }
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 13/15: Add server requirements.txt
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 8150129dcf27ca0eee1ece6febf0259b49d1b882
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:24:32 2020 +0200
Add server requirements.txt
---
server/requirements.txt | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/server/requirements.txt b/server/requirements.txt
new file mode 100644
index 0000000..04e56be
--- /dev/null
+++ b/server/requirements.txt
@@ -0,0 +1,10 @@
+aiohttp~=3.6.2
+PyYAML~=5.3.1
+multipart~=0.2.1
+elasticsearch-dsl>=7.0.0,<8.0.0
+elasticsearch~=7.8.1
+certifi~=2020.6.20
+chardet~=3.0.4
+netaddr~=0.8.0
+formatflowed~=2.0.0
+requests~=2.24.0
\ No newline at end of file
[incubator-ponymail-foal] 03/15: Add minimal server package for UI
backend
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit bcd7e32dab9735255a25ed9456ac761e095f1c86
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:13:29 2020 +0200
Add minimal server package for UI backend
This doesn't have any URL endpoints yet.
This is very much a work in progress.
---
server/main.py | 175 ++++++++++++++++++++++++++++++++++
server/plugins/__init__.py | 1 +
server/plugins/aaa.py | 29 ++++++
server/plugins/background.py | 204 ++++++++++++++++++++++++++++++++++++++++
server/plugins/configuration.py | 57 +++++++++++
server/plugins/database.py | 73 ++++++++++++++
server/plugins/formdata.py | 65 +++++++++++++
server/plugins/offloader.py | 43 +++++++++
server/plugins/server.py | 25 +++++
server/plugins/session.py | 111 ++++++++++++++++++++++
server/ponymail.yaml.example | 15 +++
11 files changed, 798 insertions(+)
diff --git a/server/main.py b/server/main.py
new file mode 100644
index 0000000..cec6dad
--- /dev/null
+++ b/server/main.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Apache Pony Mail, Codename Foal - A Python variant of Pony Mail"""
+import argparse
+import asyncio
+import importlib
+import json
+import os
+import sys
+import traceback
+
+import aiohttp.web
+import yaml
+
+import plugins.background
+import plugins.configuration
+import plugins.database
+import plugins.formdata
+import plugins.offloader
+import plugins.server
+import plugins.session
+
+PONYMAIL_FOAL_VERSION = "0.1.0"
+
+
+class Server(plugins.server.BaseServer):
+ """Main server class, responsible for handling requests and scheduling offloader threads """
+
+ def __init__(self, args: argparse.Namespace):
+ print(
+ "==== Apache Pony Mail (Foal v/%s) starting... ====" % PONYMAIL_FOAL_VERSION
+ )
+ # Load configuration
+ yml = yaml.safe_load(open(args.config))
+ self.config = plugins.configuration.Configuration(yml)
+ self.data = plugins.configuration.InterData()
+ self.handlers = dict()
+ self.dbpool = asyncio.Queue()
+ self.runners = plugins.offloader.ExecutorPool(threads=10)
+ self.server = None
+
+ # Make a pool of 15 database connections for async queries
+ for n in range(1, 15):
+ self.dbpool.put_nowait(plugins.database.Database(self.config.database))
+
+ # Load each URL endpoint
+ for endpoint_file in os.listdir("endpoints"):
+ if endpoint_file.endswith(".py"):
+ endpoint = endpoint_file[:-3]
+ m = importlib.import_module(f"endpoints.{endpoint}")
+ if hasattr(m, "register"):
+ self.handlers[endpoint] = m.__getattribute__("register")(self)
+ print(f"Registered endpoint /api/{endpoint}")
+ else:
+ print(
+ f"Could not find entry point 'register()' in {endpoint_file}, skipping!"
+ )
+
+ async def handle_request(
+ self, request: aiohttp.web.BaseRequest
+ ) -> aiohttp.web.Response:
+ """Generic handler for all incoming HTTP requests"""
+ resp: aiohttp.web.Response
+
+ # Define response headers first...
+ headers = {
+ "Server": "PyPony/%s" % PONYMAIL_FOAL_VERSION,
+ }
+
+ # Figure out who is going to handle this request, if any
+ # We are backwards compatible with the old Lua interface URLs
+ body_type = "form"
+ handler = request.path.split("/")[-1]
+ if handler.endswith(".lua"):
+ body_type = "form"
+ handler = handler[:-4]
+ if handler.endswith(".json"):
+ body_type = "json"
+ handler = handler[:-5]
+
+ # Parse form data if any
+ try:
+ indata = await plugins.formdata.parse_formdata(body_type, request)
+ except ValueError as e:
+ return aiohttp.web.Response(headers=headers, status=400, text=str(e))
+
+ # Find a handler, or 404
+ if handler in self.handlers:
+ session = await plugins.session.get_session(self, request)
+ try:
+ # Wait for endpoint response. This is typically JSON in case of success,
+ # but could be an exception (that needs a traceback) OR
+ # it could be a custom response, which we just pass along to the client.
+ output = await self.handlers[handler].exec(self, session, indata)
+ if session.database:
+ self.dbpool.put_nowait(session.database)
+ self.dbpool.task_done()
+ session.database = None
+ headers["content-type"] = "application/json"
+ if output and not isinstance(output, aiohttp.web.Response):
+ jsout = await self.runners.run(json.dumps, output, indent=2)
+ headers["Content-Length"] = str(len(jsout))
+ return aiohttp.web.Response(headers=headers, status=200, text=jsout)
+ elif isinstance(output, aiohttp.web.Response):
+ return output
+ else:
+ return aiohttp.web.Response(
+ headers=headers, status=404, text="Content not found"
+ )
+ except:
+ if session.database:
+ self.dbpool.put_nowait(session.database)
+ self.dbpool.task_done()
+ session.database = None
+ exc_type, exc_value, exc_traceback = sys.exc_info()
+ err = "\n".join(
+ traceback.format_exception(exc_type, exc_value, exc_traceback)
+ )
+ return aiohttp.web.Response(
+ headers=headers, status=500, text="API error occurred: " + err
+ )
+ else:
+ return aiohttp.web.Response(
+ headers=headers, status=404, text="API Endpoint not found!"
+ )
+
+ async def server_loop(self, loop: asyncio.AbstractEventLoop): # Note, loop never used.
+ self.server = aiohttp.web.Server(self.handle_request)
+ runner = aiohttp.web.ServerRunner(self.server)
+ await runner.setup()
+ site = aiohttp.web.TCPSite(
+ runner, self.config.server.ip, self.config.server.port
+ )
+ await site.start()
+ print(
+ "==== Serving up Pony goodness at %s:%s ===="
+ % (self.config.server.ip, self.config.server.port)
+ )
+ await plugins.background.run_tasks(self)
+
+ def run(self):
+ loop = asyncio.get_event_loop()
+ try:
+ loop.run_until_complete(self.server_loop(loop))
+ except KeyboardInterrupt:
+ pass
+ loop.close()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--config",
+ help="Configuration file to load (default: ponymail.yaml)",
+ default="ponymail.yaml",
+ )
+ cliargs = parser.parse_args()
+ pubsub_server = Server(cliargs)
+ pubsub_server.run()
diff --git a/server/plugins/__init__.py b/server/plugins/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/server/plugins/__init__.py
@@ -0,0 +1 @@
+
diff --git a/server/plugins/aaa.py b/server/plugins/aaa.py
new file mode 100644
index 0000000..d2722a2
--- /dev/null
+++ b/server/plugins/aaa.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This is the AAA library for Pony Mail codename Foal
+It handles rights management for lists.
+"""
+
+
+def can_access_email(session, email):
+ return True
+
+
+def can_access_list(session, listid):
+ return False
diff --git a/server/plugins/background.py b/server/plugins/background.py
new file mode 100644
index 0000000..fdd6d97
--- /dev/null
+++ b/server/plugins/background.py
@@ -0,0 +1,204 @@
+import asyncio
+import datetime
+import re
+import sys
+import time
+
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.helpers import async_scan
+from elasticsearch_dsl import Search
+
+import plugins.configuration
+import plugins.server
+
+PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
+
+
+class ProgTimer:
+ start: float
+ title: str
+
+ def __init__(self, title):
+ self.title = title
+
+ async def __aenter__(self):
+ sys.stdout.write(
+ "[%s] %s..." % (datetime.datetime.now().strftime("%H:%M:%S"), self.title)
+ )
+ self.start = time.time()
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ print("Done in %.2f seconds" % (time.time() - self.start))
+
+
+async def get_lists(database: plugins.configuration.DBConfig) -> dict:
+ """
+
+ :param database: a Pony Mail database configuration
+ :return: A dictionary of all mailing lists found, and whether they are considered
+ public or private
+ """
+ lists = {}
+ client = AsyncElasticsearch(
+ [
+ {
+ "host": database.hostname,
+ "port": database.port,
+ "url_prefix": database.db_prefix,
+ "use_ssl": database.secure,
+ },
+ ]
+ )
+
+ # Fetch aggregations of all public emails
+ s = Search(using=client, index=database.db_prefix + "-mbox").query(
+ "match", private=False
+ )
+ s.aggs.bucket("per_list", "terms", field="list_raw")
+
+ res = await client.search(
+ index=database.db_prefix + "-mbox", body=s.to_dict(), size=0
+ )
+
+ for ml in res["aggregations"]["per_list"]["buckets"]:
+ list_name = ml["key"].strip("<>").replace(".", "@", 1)
+ lists[list_name] = {
+ "count": ml["doc_count"],
+ "private": False,
+ }
+
+ # Ditto, for private emails
+ s = Search(using=client, index=database.db_prefix + "-mbox").query(
+ "match", private=True
+ )
+ s.aggs.bucket("per_list", "terms", field="list_raw")
+
+ res = await client.search(
+ index=database.db_prefix + "-mbox", body=s.to_dict(), size=0
+ )
+
+ for ml in res["aggregations"]["per_list"]["buckets"]:
+ list_name = ml["key"].strip("<>").replace(".", "@", 1)
+ lists[list_name] = {
+ "count": ml["doc_count"],
+ "private": True,
+ }
+ await client.close()
+
+ return lists
+
+
+async def get_public_activity(database: plugins.configuration.DBConfig) -> dict:
+ """
+
+ :param database: a PyPony database configuration
+ :return: A dictionary with activity stats
+ """
+ client = AsyncElasticsearch(
+ [
+ {
+ "host": database.hostname,
+ "port": database.port,
+ "url_prefix": database.db_prefix,
+ "use_ssl": database.secure,
+ },
+ ]
+ )
+
+ # Fetch aggregations of all public emails
+ s = (
+ Search(using=client, index=database.db_prefix + "-mbox")
+ .query("match", private=False)
+ .filter("range", date={"lt": "now+1d", "gt": "now-14d"})
+ )
+
+ s.aggs.bucket("number_of_lists", "cardinality", field="list_raw")
+ s.aggs.bucket("number_of_senders", "cardinality", field="from_raw")
+ s.aggs.bucket(
+ "daily_emails", "date_histogram", field="date", calendar_interval="1d"
+ )
+
+ res = await client.search(
+ index=database.db_prefix + "-mbox", body=s.to_dict(), size=0
+ )
+
+ no_emails = res["hits"]["total"]["value"]
+ no_lists = res["aggregations"]["number_of_lists"]["value"]
+ no_senders = res["aggregations"]["number_of_senders"]["value"]
+ daily_emails = []
+ for entry in res["aggregations"]["daily_emails"]["buckets"]:
+ daily_emails.append((entry["key"], entry["doc_count"]))
+
+ # Now the nitty gritty thread count
+ seen_emails = {}
+ seen_topics = []
+ thread_count = 0
+
+ s = (
+ Search(using=client, index=database.db_prefix + "-mbox")
+ .query("match", private=False)
+ .filter("range", date={"lt": "now+1d", "gt": "now-14d"})
+ )
+ async for doc in async_scan(
+ index=database.db_prefix + "-mbox",
+ client=client,
+ query=s.to_dict(),
+ _source_includes=[
+ "message-id",
+ "in-reply-to",
+ "subject",
+ "references",
+ "epoch",
+ "list_raw",
+ ],
+ ):
+
+ found = False
+ message_id = doc["_source"].get("message-id")
+ irt = doc["_source"].get("in-reply-to")
+ references = doc["_source"].get("references")
+ list_raw = doc["_source"].get("list_raw", "_")
+ subject = doc["_source"].get("subject", "_")
+ if irt and irt in seen_emails:
+ seen_emails[message_id] = irt
+ found = True
+ elif references:
+ for refid in re.split(r"\s+", references):
+ if refid in seen_emails:
+ seen_emails[message_id] = refid
+ found = True
+ if not found:
+ subject = PYPONY_RE_PREFIX.sub("", subject)
+ subject += list_raw
+ if subject in seen_topics:
+ seen_emails[message_id] = subject
+ else:
+ seen_topics.append(subject)
+ thread_count += 1
+
+ await client.close()
+
+ activity = {
+ "hits": no_emails,
+ "no_threads": thread_count,
+ "no_active_lists": no_lists,
+ "participants": no_senders,
+ "activity": daily_emails,
+ }
+
+ return activity
+
+
+async def run_tasks(server: plugins.server.BaseServer):
+ """
+ Runs long-lived background data gathering tasks such as gathering statistics about email activity and the list
+ of archived mailing lists, for populating the pony mail main index.
+
+ Generally runs every 2½ minutes, or whatever is set in tasks/refresh_rate in ponymail.yaml
+ """
+ while True:
+ async with ProgTimer("Gathering list of archived mailing lists"):
+ server.data.lists = await get_lists(server.config.database)
+ async with ProgTimer("Gathering bi-weekly activity stats"):
+ server.data.activity = await get_public_activity(server.config.database)
+ await asyncio.sleep(server.config.tasks.refresh_rate)
diff --git a/server/plugins/configuration.py b/server/plugins/configuration.py
new file mode 100644
index 0000000..65e5db4
--- /dev/null
+++ b/server/plugins/configuration.py
@@ -0,0 +1,57 @@
+class ServerConfig:
+ port: int
+ ip: str
+
+ def __init__(self, subyaml: dict):
+ self.ip = subyaml.get("bind", "0.0.0.0")
+ self.port = int(subyaml.get("port", 8080))
+
+
+class TaskConfig:
+ refresh_rate: int
+
+ def __init__(self, subyaml: dict):
+ self.refresh_rate = int(subyaml.get("refresh_rate", 150))
+
+
+class DBConfig:
+ hostname: str
+ port: int
+ secure: bool
+ url_prefix: str
+ db_prefix: str
+ max_hits: int
+
+ def __init__(self, subyaml: dict):
+ self.hostname = str(subyaml.get("server", "localhost"))
+ self.port = int(subyaml.get("port", 9200))
+ self.secure = bool(subyaml.get("secure", False))
+ self.url_prefix = subyaml.get("url_prefix", "")
+ self.db_prefix = str(subyaml.get("db_prefix", "ponymail"))
+ self.max_hits = int(subyaml.get("max_hits", 5000))
+
+
+class Configuration:
+ server: ServerConfig
+ database: DBConfig
+ tasks: TaskConfig
+
+ def __init__(self, yml: dict):
+ self.server = ServerConfig(yml.get("server", {}))
+ self.database = DBConfig(yml.get("database", {}))
+ self.tasks = TaskConfig(yml.get("tasks", {}))
+
+
+class InterData:
+ """
+ A mix of various global variables used throughout processes
+ """
+
+ lists: dict
+ sessions: dict
+ activity: dict
+
+ def __init__(self):
+ self.lists = {}
+ self.sessions = {}
+ self.activity = {}
diff --git a/server/plugins/database.py b/server/plugins/database.py
new file mode 100644
index 0000000..0739ee7
--- /dev/null
+++ b/server/plugins/database.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This is the Database library stub for Pony Mail codename Foal
+"""
+
+import uuid
+
+import elasticsearch
+
+import plugins.configuration
+import plugins.defuzzer
+
+
+class DBNames:
+ def __init__(self, dbprefix):
+ self.mbox = f"{dbprefix}-mbox"
+ self.source = f"{dbprefix}-source"
+ self.attachment = f"{dbprefix}-attachment"
+ self.account = f"{dbprefix}-account"
+ self.session = f"{dbprefix}-session"
+ self.notification = f"{dbprefix}-notification"
+
+
+DBError = elasticsearch.ElasticsearchException
+
+
+class Database:
+ client: elasticsearch.AsyncElasticsearch
+ config: plugins.configuration.DBConfig
+ dbs: DBNames
+
+ def __init__(self, config: plugins.configuration.DBConfig):
+ self.config = config
+ self.uuid = str(uuid.uuid4())
+ self.dbs = DBNames(config.db_prefix)
+ self.client = elasticsearch.AsyncElasticsearch(
+ [
+ {
+ "host": config.hostname,
+ "port": config.port,
+ "url_prefix": config.db_prefix,
+ "use_ssl": config.secure,
+ },
+ ]
+ )
+
+ async def search(self, index="", **kwargs):
+ if not index:
+ index = self.dbs.mbox
+ res = await self.client.search(index=index, **kwargs)
+ return res
+
+ async def get(self, index="", **kwargs):
+ if not index:
+ index = self.dbs.mbox
+ res = await self.client.get(index=index, **kwargs)
+ return res
diff --git a/server/plugins/formdata.py b/server/plugins/formdata.py
new file mode 100644
index 0000000..fe58840
--- /dev/null
+++ b/server/plugins/formdata.py
@@ -0,0 +1,65 @@
+import io
+import json
+import urllib.parse
+
+import aiohttp.web
+import multipart
+
+PYPONY_MAX_PAYLOAD = 256 * 1024
+
+
+async def parse_formdata(body_type, request: aiohttp.web.BaseRequest) -> dict:
+ indata = {}
+ for key, val in urllib.parse.parse_qsl(request.query_string):
+ indata[key] = val
+ # PUT/POST form data?
+ if request.method in ["PUT", "POST"]:
+ if request.can_read_body:
+ try:
+ if (
+ request.content_length
+ and request.content_length > PYPONY_MAX_PAYLOAD
+ ):
+ raise ValueError("Form data payload too large, max 256kb allowed")
+ body = await request.text()
+ if body_type == "json":
+ try:
+ js = json.loads(body)
+ assert isinstance(
+ js, dict
+ ) # json data MUST be an dictionary object, {...}
+ indata.update(js)
+ except ValueError:
+ raise ValueError("Erroneous payload received")
+ elif body_type == "form":
+ if (
+ request.headers.get("content-type", "").lower()
+ == "application/x-www-form-urlencoded"
+ ):
+ try:
+ for key, val in urllib.parse.parse_qsl(body):
+ indata[key] = val
+ except ValueError:
+ raise ValueError("Erroneous payload received")
+ # If multipart, turn our body into a BytesIO object and use multipart on it
+ elif (
+ "multipart/form-data"
+ in request.headers.get("content-type", "").lower()
+ ):
+ fh = request.headers.get("content-type")
+ fb = fh.find("boundary=")
+ if fb > 0:
+ boundary = fh[fb + 9 :]
+ if boundary:
+ try:
+ for part in multipart.MultipartParser(
+ io.BytesIO(body.encode("utf-8")),
+ boundary,
+ len(body),
+ ):
+ indata[part.name] = part.value
+ except ValueError:
+ raise ValueError("Erroneous payload received")
+ finally:
+ pass
+ return indata
diff --git a/server/plugins/offloader.py b/server/plugins/offloader.py
new file mode 100644
index 0000000..4a6b95e
--- /dev/null
+++ b/server/plugins/offloader.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Offloading library for pushing heavy tasks to sub threads"""
+
+import asyncio
+import concurrent.futures
+
+DEBUG = False
+
+
+class ExecutorPool:
+ def __init__(self, threads=10):
+ self.threads = concurrent.futures.ProcessPoolExecutor(max_workers=threads)
+
+ async def run(self, func, *args, **kwargs):
+ if DEBUG:
+ print("[Runner] initiating runner")
+ runner = self.threads.submit(func, *args, **kwargs)
+ if DEBUG:
+ print("[Runner] Waiting for task %r to finish" % func)
+ while runner.running():
+ await asyncio.sleep(0.01)
+ rv = runner.result()
+ if DEBUG:
+ print("[Runner] Done with task %r, put runner back in queue" % func)
+ if isinstance(rv, BaseException):
+ raise rv
+ return rv
diff --git a/server/plugins/server.py b/server/plugins/server.py
new file mode 100644
index 0000000..ae2a775
--- /dev/null
+++ b/server/plugins/server.py
@@ -0,0 +1,25 @@
+import asyncio
+import typing
+
+import aiohttp
+from elasticsearch import AsyncElasticsearch
+
+import plugins.configuration
+
+
+class Endpoint:
+ exec: typing.Callable
+
+ def __init__(self, executor):
+ self.exec = executor
+
+
+class BaseServer:
+ """Main server class, base def"""
+
+ config: plugins.configuration.Configuration
+ server: aiohttp.web.Server
+ data: plugins.configuration.InterData
+ handlers: typing.Dict[str, Endpoint]
+ database: AsyncElasticsearch
+ dbpool: asyncio.Queue
diff --git a/server/plugins/session.py b/server/plugins/session.py
new file mode 100644
index 0000000..9f6c4d3
--- /dev/null
+++ b/server/plugins/session.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is the user session handler for PyPony"""
+
+import http.cookies
+import time
+import typing
+import uuid
+
+import aiohttp.web
+
+import plugins.database
+import plugins.server
+
+PYPONY_MAX_SESSION_AGE = 86400 * 7 # Max 1 week between visits before voiding a session
+
+
+class SessionCredentials:
+ uid: str
+ name: str
+ email: str
+ provider: str
+ authoritative: bool
+ admin: bool
+
+ def __init__(self, doc: typing.Dict = None):
+ if doc:
+ pass
+ else:
+ self.uid = ""
+ self.name = ""
+ self.email = ""
+ self.provider = "generic"
+ self.authoritative = False
+ self.admin = False
+
+
+class SessionObject:
+ uid: str
+ created: int
+ last_accessed: int
+ credentials: SessionCredentials
+ database: typing.Optional[plugins.database.Database]
+
+ def __init__(self, server: plugins.server.BaseServer, doc=None):
+ self.database = None
+ if not doc:
+ now = int(time.time())
+ self.created = now
+ self.last_accessed = now
+ self.credentials = SessionCredentials()
+ self.uid = str(uuid.uuid4())
+ else:
+ self.created = doc["created"]
+ self.last_accessed = doc["last_accessed"]
+ self.credentials = SessionCredentials(doc["credentials"])
+ self.uid = doc["uid"]
+
+
+async def get_session(
+ server: plugins.server.BaseServer, request: aiohttp.web.BaseRequest
+ ) -> SessionObject:
+ session_id = None
+ session = None
+ if request.headers.get("cookie"):
+ for cookie_header in request.headers.getall("cookie"):
+ cookies: http.cookies.SimpleCookie = http.cookies.SimpleCookie(
+ cookie_header
+ )
+ if "ponymail" in cookies:
+ session_id = cookies["ponymail"].value
+ break
+
+ if session_id in server.data.sessions:
+ x_session = server.data.sessions[session_id]
+ now = int(time.time())
+ if (now - x_session.last_accessed) > PYPONY_MAX_SESSION_AGE:
+ del server.data.sessions[session_id]
+ else:
+ session = x_session
+ if not session:
+ session = SessionObject(server)
+ session.database = await server.dbpool.get()
+ return session
+
+
+async def set_session(server: plugins.server.BaseServer, **credentials):
+ """Create a new user session in the database"""
+ session_id = str(uuid.uuid4())
+ cookie: http.cookies.SimpleCookie = http.cookies.SimpleCookie()
+ cookie["ponymail"] = session_id
+ session = SessionObject(server)
+ session.credentials = SessionCredentials(credentials)
+ server.data.sessions[session_id] = session
+
+ return cookie.output()
diff --git a/server/ponymail.yaml.example b/server/ponymail.yaml.example
new file mode 100644
index 0000000..b0e06cc
--- /dev/null
+++ b/server/ponymail.yaml.example
@@ -0,0 +1,15 @@
+server:
+ port: 8080 # Port to bind to
+ bind: 127.0.0.1 # IP to bind to - typically 127.0.0.1 for localhost or 0.0.0.0 for all IPs
+
+
+database:
+ server: localhost # The hostname of the ElasticSearch database
+ port: 9200 # ES Port
+ secure: false # Whether TLS is enabled on ES
+ url_prefix: ~ # URL prefix, if proxying to ES
+ db_prefix: ponymail # DB prefix, usually 'ponymail'
+ max_hits: 15000 # Maximum number of emails to process in a search
+
+tasks:
+ refresh_rate: 150 # Background indexer run interval, in seconds
[incubator-ponymail-foal] 11/15: needs another newline
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit a724b9befacb728164c36f7348e32c3de593d30f
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:23:05 2020 +0200
needs another newline
---
server/endpoints/preferences.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/server/endpoints/preferences.py b/server/endpoints/preferences.py
index aa3a682..bfd7e3b 100644
--- a/server/endpoints/preferences.py
+++ b/server/endpoints/preferences.py
@@ -21,6 +21,7 @@ import plugins.aaa
""" Generic preferences endpoint for Pony Mail codename Foal"""
""" This is incomplete, but will work for anonymous tests. """
+
async def process(
server: plugins.server.BaseServer, session: dict, indata: dict
) -> dict:
[incubator-ponymail-foal] 06/15: add pminfo endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 74d6bd012d37c0ecb773e80383586b211d454931
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:17:58 2020 +0200
add pminfo endpoint
---
server/endpoints/pminfo.py | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/server/endpoints/pminfo.py b/server/endpoints/pminfo.py
new file mode 100644
index 0000000..21bf996
--- /dev/null
+++ b/server/endpoints/pminfo.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple endpoint that returns the server's gathered activity data"""
+
+import plugins.server
+
+
+async def process(
+ server: plugins.server.BaseServer, session: dict, indata: dict
+) -> dict:
+ return server.data.activity
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 07/15: Add stats endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 775998e7ffaec614323584e815451f1b9c88a104
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:20:07 2020 +0200
Add stats endpoint
As mentioned in the code, this only works with public emails for now.
AAA is being worked on.
---
server/endpoints/stats.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py
new file mode 100644
index 0000000..2f470f6
--- /dev/null
+++ b/server/endpoints/stats.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple endpoint that returns the server's gathered activity data"""
+""" THIS ONLY DEALS WITH PUBLIC EMAILS FOR NOW - AAA IS BEING WORKED ON"""
+import plugins.server
+import plugins.session
+import plugins.mbox
+import plugins.defuzzer
+import plugins.offloader
+import re
+import collections
+import email.utils
+import typing
+
+PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
+
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> dict:
+
+ query_defuzzed = plugins.defuzzer.defuzz(indata)
+ query_defuzzed_nodate = plugins.defuzzer.defuzz(indata, nodate=True)
+ results = await plugins.mbox.query(
+ session,
+ query_defuzzed,
+ query_limit=server.config.database.max_hits,
+ shorten=True,
+ )
+
+ for msg in results:
+ msg["gravatar"] = plugins.mbox.gravatar(msg)
+ wordcloud = await plugins.mbox.wordcloud(session, query_defuzzed)
+ first_year, last_year = await plugins.mbox.get_years(session, query_defuzzed_nodate)
+
+ tstruct, authors = await server.runners.run(plugins.mbox.construct_threads, results)
+ xlist = indata.get("list", "*")
+ xdomain = indata.get("domain", "*")
+
+ all_authors = sorted(
+ [[author, count] for author, count in authors.items()], key=lambda x: x[1]
+ )
+ top10_authors = []
+ for x in [x for x in reversed([x for x in all_authors])][:10]:
+ author, count = x
+ name, address = email.utils.parseaddr(author)
+ top10_authors.append(
+ {
+ "email": address,
+ "name": name,
+ "count": count,
+ "gravatar": plugins.mbox.gravatar(author),
+ }
+ )
+
+ return {
+ "firstYear": first_year,
+ "lastYear": last_year,
+ "hits": len(results),
+ "numparts": len(authors),
+ "no_threads": len(tstruct),
+ "emails": list(sorted(results, key=lambda x: x['epoch'])),
+ "cloud": wordcloud,
+ "participants": top10_authors,
+ "thread_struct": tstruct,
+ "search_list": f"<{xlist}.{xdomain}>",
+ "domain": xdomain,
+ "list": f"{xlist}@{xdomain}",
+ }
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 02/15: only check this if --generator is
set
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 99a4ff5ca874446c12c60bbbbeb6039e2fb217eb
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:01:22 2020 +0200
only check this if --generator is set
---
tools/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/setup.py b/tools/setup.py
index 0a9a362..0a67476 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -226,7 +226,7 @@ if args.generator:
+ "\n"
)
sys.exit(-1)
-if any(x == "dkim" for x in args.generator.split(' ')) and args.nonce is not None:
+if args.generator and any(x == "dkim" for x in args.generator.split(' ')) and args.nonce is not None:
nonce = args.nonce
if not hostname:
[incubator-ponymail-foal] 10/15: Add preferences endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 7e45a957b1d97fbbb6dea2c33d82c1a8eed449ca
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:22:48 2020 +0200
Add preferences endpoint
---
server/endpoints/preferences.py | 45 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/server/endpoints/preferences.py b/server/endpoints/preferences.py
new file mode 100644
index 0000000..aa3a682
--- /dev/null
+++ b/server/endpoints/preferences.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import plugins.server
+import plugins.aaa
+
+""" Generic preferences endpoint for Pony Mail codename Foal"""
+""" This is incomplete, but will work for anonymous tests. """
+
+async def process(
+ server: plugins.server.BaseServer, session: dict, indata: dict
+) -> dict:
+ prefs = {"login": {}}
+ lists = {}
+ for ml, entry in server.data.lists.items():
+ if "@" in ml:
+ lname, ldomain = ml.split("@", 1)
+ can_access = True
+ if entry.get("private", False):
+ can_access = plugins.aaa.can_access_list(session, ml)
+ if can_access:
+ if ldomain not in lists:
+ lists[ldomain] = {}
+ lists[ldomain][lname] = entry["count"]
+ prefs["lists"] = lists
+
+ return prefs
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 15/15: permalinks with an S
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit d5bc8d06950dd89d1e09cfd579ec305a8fadef6d
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:33:51 2020 +0200
permalinks with an S
---
server/plugins/mbox.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
index 1da3f43..c938f24 100644
--- a/server/plugins/mbox.py
+++ b/server/plugins/mbox.py
@@ -207,7 +207,7 @@ async def get_source(session: plugins.session.SessionObject, permalink: str = No
res = await session.database.search(
index=doctype,
size=1,
- body={"query": {"bool": {"must": [{"match": {"permalink": permalink}}]}}},
+ body={"query": {"bool": {"must": [{"match": {"permalinks": permalink}}]}}},
)
if len(res["hits"]["hits"]) == 1:
doc = res["hits"]["hits"][0]
[incubator-ponymail-foal] 14/15: Add a simple README for now,
with run instructions for httpd
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit eb798e43fe9692d21ab0b61ad7809176ed77f4bc
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:32:52 2020 +0200
Add a simple README for now, with run instructions for httpd
---
server/README.md | 37 +++++++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/server/README.md b/server/README.md
new file mode 100644
index 0000000..ae2e610
--- /dev/null
+++ b/server/README.md
@@ -0,0 +1,37 @@
+# Pony Mail Foal - Backend UI Server
+
+This is the (as of yet incomplete) backedn server for the Foal UI.
+While it works on all-public archives with searching, threads, emails
+and sources, the AAA (Access, Authentication and Authorization) plugin
+is not yet complete, in part due to waiting for OAuth to be completed.
+
+This backend should not yet be used for private email archives unless
+restricted behind some form of external/parent authentication mechanism.
+
+
+## How to run:
+- Install the Pony Mail service through `tools/setup.py` first.
+ This will create a ponymail.yaml for the backend server as well
+- install `pipenv`, for example via aptitude: `apt install pipenv`.
+- Install the environment for the server: `pipenv install -r requirements.txt`
+- Run the server: `pipenv run python3 main.py`
+
+This should fire up a backend server on 127.0.0.1:8080. You can then proxy to
+that using a web server of your choice. The `/api/` URL of your online archive
+should be passed straight to the backend, while the rest should be served from
+the `webui/` directory in this repository.
+
+An example Apache HTTPd configuration could be (for plain-text HTTP):
+
+```
+<VirtualHost *:80>
+ ServerName archives.example.com
+ ServerAdmin webmaster@localhost
+ DocumentRoot /var/www/foal/webui/
+ ProxyPass /api/ http://localhost:8080/api/
+ <Directory /var/www/foal/webui/>
+ Require all granted
+ </Directory>
+</VirtualHost>
+```
+
[incubator-ponymail-foal] 05/15: add mbox utulity plugin
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit a639cb0687862e5d3ce209820ce1ed6161d052ff
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:16:48 2020 +0200
add mbox utulity plugin
---
server/plugins/mbox.py | 432 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 432 insertions(+)
diff --git a/server/plugins/mbox.py b/server/plugins/mbox.py
new file mode 100644
index 0000000..1da3f43
--- /dev/null
+++ b/server/plugins/mbox.py
@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This is the mbox library for Pony Mail.
+It handles fetching (the right) emails for
+pages that need it.
+"""
+
+
+import base64
+import binascii
+import datetime
+import email.utils
+import hashlib
+# Main imports
+import re
+import typing
+
+from elasticsearch.helpers import async_scan
+
+import plugins.aaa
+import plugins.session
+
+PYPONY_RE_PREFIX = re.compile(r"^([a-zA-Z]+:\s*)+")
+
+mbox_cache_privacy = {}
+
+
+def extract_name(addr):
+ """ Extract name and email from from: header """
+ m = re.match(r"^([^<]+)\s*<(.+)>$", addr)
+ if m:
+ return [m.group(1), m.group(2)]
+ else:
+ addr = addr.strip("<>")
+ return [addr, addr]
+
+
+def anonymize(doc):
+ """ Anonymizes an email, hiding author email addresses."""
+ # ES direct hit?
+ ptr: typing.Dict[str, str] = doc
+ if "_source" in doc:
+ ptr = doc["_source"]
+
+ if "from" in ptr:
+ frname, fremail = extract_name(ptr["from"])
+ ptr["md5"] = hashlib.md5(
+ bytes(fremail.lower(), encoding="ascii", errors="replace")
+ ).hexdigest()
+ ptr["from"] = re.sub(
+ r"<(\S{1,2})\S*@([-a-zA-Z0-9_.]+)>", "<\\1..@\\2>", ptr["from"]
+ )
+ if ptr["body"]:
+ ptr["body"] = re.sub(
+ r"<(\S{1,2})\S*@([-a-zA-Z0-9_.]+)>", "<\\1..@\\2>", ptr["body"]
+ )
+ return doc
+
+
+async def find_parent(session, doc: typing.Dict[str, str]):
+ """
+ Locates the first email in a thread by going back through all the
+ in-reply-to headers and finding their source.
+ """
+ step = 0
+ # max 50 steps up in the hierarchy
+ while step < 50:
+ step = step + 1
+ irt: str = doc["in-reply-to"] if "in-reply-to" in doc else None
+ if not irt:
+ break # Shouldn't happen because irt is always present currently
+ # Extract the reference, if any
+ m = re.search(r"(<[^>]+>)", irt)
+ if not m:
+ break
+ ref = m.group(1)
+ newdoc = await get_email(session, messageid=ref)
+ # Did we find something, and can the user access it?
+ if not newdoc or not plugins.aaa.can_access_email(session, newdoc):
+ break
+ else:
+ doc = newdoc
+ return doc
+
+
+async def fetch_children(session, pdoc, counter=0, pdocs=None, short=False):
+ """
+ Fetches all child messages of a parent email
+ """
+ if pdocs is None:
+ pdocs = {}
+ counter = counter + 1
+ if counter > 250:
+ return []
+ docs = await get_email(session, irt=pdoc["message-id"])
+
+ thread = []
+ emails = []
+ for doc in docs:
+ # Make sure email is accessible
+ if plugins.aaa.can_access_email(session, doc):
+ if doc["mid"] not in pdocs:
+ mykids, myemails, pdocs = await fetch_children(
+ session, doc, counter, pdocs, short=short
+ )
+ if short:
+ xdoc = {
+ "tid": doc["mid"],
+ "mid": doc["mid"],
+ "message-id": doc["message-id"],
+ "subject": doc["subject"],
+ "from": doc["from"],
+ "id": doc["mid"],
+ "epoch": doc["epoch"],
+ "children": mykids,
+ "irt": doc["in-reply-to"],
+ "list_raw": doc["list_raw"],
+ }
+ thread.append(xdoc)
+ pdocs[doc["mid"]] = xdoc
+ else:
+ thread.append(doc)
+ pdocs[doc["mid"]] = doc
+ for kid in mykids:
+ if kid["mid"] not in pdocs:
+ pdocs[kid["mid"]] = kid
+ emails.append(kid)
+ return thread, emails, pdocs
+
+
+async def get_email(
+ session: plugins.session.SessionObject,
+ permalink: str = None,
+ messageid=None,
+ irt=None,
+ source=False,
+):
+ doctype = session.database.dbs.mbox
+ if source:
+ doctype = session.database.dbs.source
+ # Older indexes may need a match instead of a strict terms agg in order to find
+ # emails in DBs that may have been incorrectly analyzed.
+ aggtype = "match"
+ if permalink:
+ try:
+ doc = await session.database.get(index=doctype, id=permalink)
+ if doc and plugins.aaa.can_access_email(session, doc):
+ if not session.credentials:
+ doc = anonymize(doc)
+ doc["_source"]["id"] = doc["_source"]["mid"]
+ return doc["_source"]
+ except session.database.DBError:
+ pass
+ elif messageid:
+ res = await session.database.search(
+ index=doctype,
+ size=1,
+ body={"query": {"bool": {"must": [{aggtype: {"message-id": messageid}}]}}},
+ )
+ if len(res["hits"]["hits"]) == 1:
+ doc = res["hits"]["hits"][0]["_source"]
+ doc["id"] = doc["mid"]
+ if plugins.aaa.can_access_email(session, doc):
+ if not session.credentials:
+ doc = anonymize(doc)
+ return doc
+ elif irt:
+ res = await session.database.search(
+ index=doctype,
+ size=250,
+ body={"query": {"bool": {"must": [{aggtype: {"in-reply-to": irt}}]}}},
+ )
+ docs = []
+ for doc in res["hits"]["hits"]:
+ if plugins.aaa.can_access_email(session, doc):
+ if not session.credentials:
+ doc = anonymize(doc)
+ doc["_source"]["id"] = doc["_source"]["mid"]
+ docs.append(doc["_source"])
+ return docs
+ return None
+
+
+async def get_source(session: plugins.session.SessionObject, permalink: str = None):
+ doctype = session.database.dbs.source
+ try:
+ doc = await session.database.get(index=doctype, id=permalink)
+ return doc
+ except session.database.DBError:
+ pass
+ res = await session.database.search(
+ index=doctype,
+ size=1,
+ body={"query": {"bool": {"must": [{"match": {"permalink": permalink}}]}}},
+ )
+ if len(res["hits"]["hits"]) == 1:
+ doc = res["hits"]["hits"][0]
+ doc["id"] = doc["_id"]
+ # Check for base64-encoded source
+ if ':' not in doc['_source']['source']:
+ try:
+ doc['_source']['source'] = base64.standard_b64decode(doc['_source']['source'])\
+ .decode('utf-8', 'replace')
+ except binascii.Error:
+ pass # If it wasn't base64 after all, just return as is
+ return doc
+ return None
+
+
+def get_list(session, listid, fr=None, to=None, limit=10000):
+ """
+ Loads emails from a specified list.
+ If fr and to are not specified, loads the last 30 days.
+ """
+ res = session.DB.ES.search(
+ index=session.DB.dbs.mbox,
+ size=limit,
+ body={
+ "query": {"bool": {"must": [{"term": {"list_raw": listid}}]}},
+ "sort": [{"epoch": {"order": "asc"}}],
+ },
+ )
+ docs = []
+ for hit in res["hits"]["hits"]:
+ doc = hit["_source"]
+ if plugins.aaa.can_access_email(session, doc):
+ if not session.user:
+ doc = anonymize(doc)
+ docs.append(doc)
+ return docs
+
+
+async def query(
+ session: plugins.session.SessionObject,
+ query_defuzzed,
+ query_limit=10000,
+ shorten=False,
+):
+ """
+ Advanced query and grab for stats.py
+ """
+ docs = []
+ hits = 0
+ async for hit in async_scan(
+ client=session.database.client,
+ query={
+ "query": {"bool": query_defuzzed},
+ "sort": [{"epoch": {"order": "asc"}}],
+ },
+ ):
+ doc = hit["_source"]
+ doc["id"] = doc["mid"]
+ if plugins.aaa.can_access_email(session, doc):
+ if not session.credentials:
+ doc = anonymize(doc)
+ if shorten:
+ doc["body"] = (doc["body"] or "")[:200]
+ docs.append(doc)
+ hits += 1
+ if hits > query_limit:
+ break
+ return docs
+
+
+async def wordcloud(session, query_defuzzed):
+ """
+ Wordclouds via significant terms query in ES
+ """
+ wc = {}
+ res = await session.database.search(
+ body={
+ "size": 0,
+ "query": {"bool": query_defuzzed},
+ "aggregations": {
+ "cloud": {"significant_terms": {"field": "subject", "size": 10}}
+ },
+ }
+ )
+
+ for hit in res["aggregations"]["cloud"]["buckets"]:
+ wc[hit["key"]] = hit["doc_count"]
+
+ return wc
+
+
+def is_public(session: plugins.session.SessionObject, listname):
+ """ Quickly determine if a list if fully public, private or mixed """
+ if "@" not in listname:
+ lname, ldomain = listname.strip("<>").split(".", 1)
+ listname = f"{lname}@{ldomain}"
+ if listname in session.server.data.lists:
+ return not session.server.data.lists[listname]["private"]
+ else:
+ return False # Default to not public
+
+
+async def get_list_stats(session, maxage="90d", admin=False):
+ """
+ Fetches a list of all mailing lists available (and visible).
+ Use the admin flag to override AAA and see everything
+ """
+ daterange = {"gt": "now-%s" % maxage, "lt": "now+1d"}
+ res = session.database.search(
+ index=session.DB.dbs.mbox,
+ size=0,
+ body={
+ "query": {"bool": {"must": [{"range": {"date": daterange}}]}},
+ "aggs": {"listnames": {"terms": {"field": "list_raw", "size": 10000}}},
+ },
+ )
+ lists = {}
+ for entry in res["aggregations"]["listnames"]["buckets"]:
+
+ # Normalize list name
+ listname = entry["key"].lower().strip("<>")
+
+ # Check access
+ if (
+ is_public(session, listname)
+ or admin
+ or plugins.aaa.canViewList(session, listname)
+ ):
+ # Change foo.bar.baz to foo@bar.baz
+ listname = listname.replace(".", "@", 1)
+ lists[listname] = entry["doc_count"]
+ return lists
+
+
+async def get_years(session, query_defuzzed):
+ """ Fetches the oldest and youngest email, returns the years between them """
+
+ # Get oldest doc
+ res = await session.database.search(
+ index=session.database.dbs.mbox,
+ size=1,
+ body={"query": {"bool": query_defuzzed}, "sort": {"epoch": "asc"}},
+ )
+ oldest = 1970
+ if res["hits"]["hits"]:
+ doc = res["hits"]["hits"][0]
+ oldest = datetime.datetime.fromtimestamp(doc["_source"]["epoch"]).year
+
+ # Get youngest doc
+ res = await session.database.search(
+ size=1, body={"query": {"bool": query_defuzzed}, "sort": {"epoch": "desc"}}
+ )
+ youngest = 1970
+ if res["hits"]["hits"]:
+ doc = res["hits"]["hits"][0]
+ youngest = datetime.datetime.fromtimestamp(doc["_source"]["epoch"]).year
+
+ return oldest, youngest
+
+
+def find_root_subject(threads, hashdict, root_email, osubject=None):
+ """Finds the discussion origin of an email, if present"""
+ irt = root_email.get("in-reply-to")
+ subject = root_email.get("subject")
+ subject = subject.replace("\n", "") # Crop multi-line subjects
+
+ # First, the obvious - look for an in-reply-to in our existing dict with a matching subject
+ if irt and irt in hashdict:
+ if hashdict[irt].get("subject") == subject:
+ return hashdict[irt]
+
+ # If that failed, we break apart our subject
+ if osubject:
+ rsubject = osubject
+ else:
+ rsubject = PYPONY_RE_PREFIX.sub("", subject) + "_" + root_email.get("list_raw")
+ for thread in threads:
+ if thread.get("tsubject") == rsubject:
+ return thread
+
+ return None
+
+
+def construct_threads(emails: typing.List[typing.Dict]):
+ """Turns a list of emails into a nested thread structure"""
+ threads = []
+ authors = {}
+ hashdict = {}
+ for cur_email in sorted(emails, key=lambda x: x["epoch"]):
+ author = cur_email.get("from")
+ if author not in authors:
+ authors[author] = 0
+ authors[author] += 1
+ subject = cur_email.get("subject").replace("\n", "") # Crop multi-line subjects
+ tsubject = PYPONY_RE_PREFIX.sub("", subject) + "_" + cur_email.get("list_raw")
+ parent = find_root_subject(threads, hashdict, cur_email, tsubject)
+ xemail = {
+ "children": [],
+ "tid": cur_email.get("mid"),
+ "subject": subject,
+ "tsubject": tsubject,
+ "epoch": cur_email.get("epoch"),
+ "nest": 1,
+ }
+ if not parent:
+ threads.append(xemail)
+ else:
+ xemail["nest"] = parent["nest"] + 1
+ parent["children"].append(xemail)
+ hashdict[cur_email.get("message-id", "??")] = xemail
+ return threads, authors
+
+
+def gravatar(eml):
+ """Generates a gravatar hash from an email address"""
+ if isinstance(eml, str):
+ header_from = eml
+ else:
+ header_from = eml.get("from", "??")
+ mailaddr = email.utils.parseaddr(header_from)[1]
+ ghash = hashlib.md5(mailaddr.encode("utf-8")).hexdigest()
+ return ghash
[incubator-ponymail-foal] 01/15: add backend ui config
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 0f1210f3ce08caab4f8ee7420c3709e5dd373353
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:00:26 2020 +0200
add backend ui config
---
tools/setup.py | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/tools/setup.py b/tools/setup.py
index 543b220..0a9a362 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -425,6 +425,26 @@ if not os.path.exists("../site/js/config.js") and os.path.exists(
):
shutil.copy("../site/js/config.js.sample", "../site/js/config.js")
+print("Writing UI backend configuration file ponymail.yaml")
+with open("../server/ponymail.yaml", "w") as f:
+ f.write("""
+server:
+ port: 8080 # Port to bind to
+ bind: 127.0.0.1 # IP to bind to - typically 127.0.0.1 for localhost or 0.0.0.0 for all IPs
+
+
+database:
+ server: %s # The hostname of the ElasticSearch database
+ port: %u # ES Port
+ secure: false # Whether TLS is enabled on ES
+ url_prefix: ~ # URL prefix, if proxying to ES
+ db_prefix: %s # DB prefix, usually 'ponymail'
+ max_hits: 15000 # Maximum number of emails to process in a search
+
+tasks:
+ refresh_rate: 150 # Background indexer run interval, in seconds
+""" % (hostname, port, dbname))
+
print("All done, Pony Mail should...work now :)")
print(
[incubator-ponymail-foal] 09/15: Add source endpoint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 6a71ff0c7354c1419457c2bf6b7637a5924facd9
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Sun Sep 6 19:21:18 2020 +0200
Add source endpoint
---
server/endpoints/source.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 47 insertions(+)
diff --git a/server/endpoints/source.py b/server/endpoints/source.py
new file mode 100644
index 0000000..46d417c
--- /dev/null
+++ b/server/endpoints/source.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Simple endpoint that returns an email or an attachment from one"""
+
+import plugins.server
+import plugins.session
+import plugins.mbox
+import aiohttp.web
+import plugins.aaa
+
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> aiohttp.web.Response:
+
+ email = await plugins.mbox.get_email(session, id=indata.get("id"))
+ if email and isinstance(email, dict):
+ if plugins.aaa.can_access_email(session, email):
+ source = await plugins.mbox.get_source(session, permalink=email["mid"])
+ if source:
+ return aiohttp.web.Response(
+ headers={"Content-Type": "text/plain"},
+ status=200,
+ text=source["_source"]["source"],
+ )
+ return aiohttp.web.Response(headers={}, status=404, text="Email not found")
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)