You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2022/01/24 22:53:13 UTC
[incubator-ponymail-foal] branch master updated (b066163 -> 71754e1)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from b066163 regen JS
new bbd540b Add plain html indexer for easing search engine indexing by certain engines
new 71754e1 bump version
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/endpoints/plain.py | 167 ++++++++++++++++++++++++++++++++++++++++++++++
server/server_version.py | 2 +-
2 files changed, 168 insertions(+), 1 deletion(-)
create mode 100644 server/endpoints/plain.py
Re: [incubator-ponymail-foal] 01/02: Add plain html indexer for easing search engine indexing by certain engines
Posted by sebb <se...@gmail.com>.
On Mon, 24 Jan 2022 at 22:53, <hu...@apache.org> wrote:
>
> This is an automated email from the ASF dual-hosted git repository.
>
> humbedooh pushed a commit to branch master
> in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
>
> commit bbd540b58c7fb4b4ca2974675182016c1cf135b4
> Author: Daniel Gruno <hu...@apache.org>
> AuthorDate: Mon Jan 24 23:52:25 2022 +0100
>
> Add plain html indexer for easing search engine indexing by certain engines
This needs documenting.
> ---
> server/endpoints/plain.py | 167 ++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 167 insertions(+)
>
> diff --git a/server/endpoints/plain.py b/server/endpoints/plain.py
> new file mode 100644
> index 0000000..14d2f9b
> --- /dev/null
> +++ b/server/endpoints/plain.py
> @@ -0,0 +1,167 @@
> +#!/usr/bin/env python3
> +# -*- coding: utf-8 -*-
> +# Licensed to the Apache Software Foundation (ASF) under one or more
> +# contributor license agreements. See the NOTICE file distributed with
> +# this work for additional information regarding copyright ownership.
> +# The ASF licenses this file to You under the Apache License, Version 2.0
> +# (the "License"); you may not use this file except in compliance with
> +# the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +"""Plain text endpoint for enabling some search engines to index mail archives"""
> +
> +import plugins.server
> +import plugins.session
> +import plugins.messages
> +import aiohttp.web
> +import html
> +
> +
> +def count_replies(thread):
> + """Simple function for counting how many replies an email thread has"""
> + count = 0
> + for child in thread["children"]:
> + count += count_replies(child) + 1
> + return count
> +
> +
> +async def process(
> + server: plugins.server.BaseServer,
> + session: plugins.session.SessionObject,
> + indata: dict,
> +) -> aiohttp.web.Response:
> +
> + output = ""
> + canonical_link = None
> + title = "Apache Pony Mail"
> +
> + # Has a list or thread id been provided?
> + list_id = html.escape(indata.get("list", ""))
> + thread_id = html.escape(indata.get("thread", ""))
> +
> + # Show an email (or thread)
> + if thread_id:
> + canonical_link = f"""/thread.html/{thread_id}"""
> + email = await plugins.messages.get_email(session, permalink=thread_id)
> + if email:
> + listname = html.escape(
> + "@".join(email.get("list_raw", "").strip("<>").split(".", 1))
> + )
> + date = html.escape(email.get("date", ""))
> + author = html.escape(email.get("from"))
> + output += f"""Posted to <a href="/list.html?{listname}">{listname}</a> by {author} on {date} UTC<br/>"""
> + title = html.escape(email.get("subject", ""))
> + body = html.escape(email.get("body", ""))
> + thread, emails, _pdocs = await plugins.messages.fetch_children(
> + session, email
> + )
> + output += f"""<h1>{email["subject"]}</h1><pre>{body}</pre><hr/>\n"""
> + for tid, email in _pdocs.items():
> + body = html.escape(email.get("body", ""))
> + author = html.escape(email.get("from"))
> + output += f"""<h2>{email["subject"]}</h2>\n<b>Posted by {author}.</b><hr/><pre>{body}</pre><hr/>\n"""
> + # Show a list
> + elif list_id:
> + # Make sure we can actually index this list
> + can_view = False
> + if list_id in server.data.lists:
> + if not server.data.lists[list_id].get("private", True):
> + can_view = True
> + if can_view:
> + l, d = list_id.split("@", 1)
> + month = indata.get("date")
> + mydata = {
> + "list": l,
> + "domain": d,
> + }
> +
> + # Do we have a specific month to show?
> + if month:
> + title = html.escape(f"{list_id}, {month}")
> + mydata["date"] = month
> + query_defuzzed = plugins.defuzzer.defuzz(mydata)
> + canonical_link = f"/list.html?{list_id}:{month}"
> + results = await plugins.messages.query(
> + session,
> + query_defuzzed,
> + query_limit=server.config.database.max_hits,
> + )
> + threads = plugins.messages.ThreadConstructor(results)
> + thread_struct, authors = await server.runners.run(threads.construct)
> + for (
> + thread
> + ) in (
> + thread_struct
> + ): # Make a list item for each thread (not for each email)
> + author = "Unknown"
> + date = "Unknown"
> + count = count_replies(thread)
> + # Find the email in the results pile and grab author and date
> + for k in results:
> + if k["id"] == thread["tid"]:
> + author = html.escape(k["from"])
> + date = html.escape(k["date"])
> + break
> + output += f"""- <a href="?thread={thread["tid"]}">{thread["subject"]}</a> - posted by {author} on {date} UTC, {count} replies.<br/>\n"""
> + # No month specified, which means just show all months with email in 'em
> + else:
> + title = list_id
> + canonical_link = f"/list.html?{list_id}"
> + output = f"""<link rel="canonical" href="/list.html?{list_id}" />\n"""
> + query_defuzzed_nodate = plugins.defuzzer.defuzz(mydata, nodate=True)
> + (
> + oldest,
> + youngest,
> + active_months,
> + ) = await plugins.messages.get_activity_span(
> + session, query_defuzzed_nodate
> + )
> + for month, activity in active_months.items():
> + output += (
> + f"""<a href="?list={list_id}&date={month}">{month}</a><br/>"""
> + )
> + else: # Just list all lists?
> + canonical_link = "/"
> + output = f"""<link rel="canonical" href="/" />\n"""
> + # Sort by domain, then by list name
> + for ml in sorted(server.data.lists.keys(), key=lambda x: x.split("@", 1)[-1] + "-" + x.split("@", 1)[0]):
> + entry = server.data.lists[ml]
> + if "@" in ml:
> + if not entry.get("private", True): # Only index public lists
> + output += f"<a href='?list={ml}'>{ml}</a><br/>\n"
> +
> + if output and canonical_link:
> + output_interpolated = f"""
> + <html>
> + <head>
> + <link rel="canonical" href="{canonical_link}" />
> + <title>{title}</title>
> + </head>
> + <body>
> + <i>You are viewing a plain text version of this content. The canonical link for it is <a href="{canonical_link}">here</a>.</i><hr/>
> + {output}
> + </body>
> + </html>
> + """
> + return aiohttp.web.Response(
> + headers={"Content-Type": "text/html; charset=utf-8"},
> + status=200,
> + text=output_interpolated,
> + )
> + else:
> + return aiohttp.web.Response(
> + headers={"Content-Type": "text/plain"},
> + status=200,
> + text="No data",
> + )
> +
> +
> +def register(server: plugins.server.BaseServer):
> + return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 01/02: Add plain html indexer for easing search engine indexing by certain engines
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit bbd540b58c7fb4b4ca2974675182016c1cf135b4
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Jan 24 23:52:25 2022 +0100
Add plain html indexer for easing search engine indexing by certain engines
---
server/endpoints/plain.py | 167 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 167 insertions(+)
diff --git a/server/endpoints/plain.py b/server/endpoints/plain.py
new file mode 100644
index 0000000..14d2f9b
--- /dev/null
+++ b/server/endpoints/plain.py
@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plain text endpoint for enabling some search engines to index mail archives"""
+
+import plugins.server
+import plugins.session
+import plugins.messages
+import aiohttp.web
+import html
+
+
+def count_replies(thread):
+ """Simple function for counting how many replies an email thread has"""
+ count = 0
+ for child in thread["children"]:
+ count += count_replies(child) + 1
+ return count
+
+
+async def process(
+ server: plugins.server.BaseServer,
+ session: plugins.session.SessionObject,
+ indata: dict,
+) -> aiohttp.web.Response:
+
+ output = ""
+ canonical_link = None
+ title = "Apache Pony Mail"
+
+ # Has a list or thread id been provided?
+ list_id = html.escape(indata.get("list", ""))
+ thread_id = html.escape(indata.get("thread", ""))
+
+ # Show an email (or thread)
+ if thread_id:
+ canonical_link = f"""/thread.html/{thread_id}"""
+ email = await plugins.messages.get_email(session, permalink=thread_id)
+ if email:
+ listname = html.escape(
+ "@".join(email.get("list_raw", "").strip("<>").split(".", 1))
+ )
+ date = html.escape(email.get("date", ""))
+ author = html.escape(email.get("from"))
+ output += f"""Posted to <a href="/list.html?{listname}">{listname}</a> by {author} on {date} UTC<br/>"""
+ title = html.escape(email.get("subject", ""))
+ body = html.escape(email.get("body", ""))
+ thread, emails, _pdocs = await plugins.messages.fetch_children(
+ session, email
+ )
+ output += f"""<h1>{email["subject"]}</h1><pre>{body}</pre><hr/>\n"""
+ for tid, email in _pdocs.items():
+ body = html.escape(email.get("body", ""))
+ author = html.escape(email.get("from"))
+ output += f"""<h2>{email["subject"]}</h2>\n<b>Posted by {author}.</b><hr/><pre>{body}</pre><hr/>\n"""
+ # Show a list
+ elif list_id:
+ # Make sure we can actually index this list
+ can_view = False
+ if list_id in server.data.lists:
+ if not server.data.lists[list_id].get("private", True):
+ can_view = True
+ if can_view:
+ l, d = list_id.split("@", 1)
+ month = indata.get("date")
+ mydata = {
+ "list": l,
+ "domain": d,
+ }
+
+ # Do we have a specific month to show?
+ if month:
+ title = html.escape(f"{list_id}, {month}")
+ mydata["date"] = month
+ query_defuzzed = plugins.defuzzer.defuzz(mydata)
+ canonical_link = f"/list.html?{list_id}:{month}"
+ results = await plugins.messages.query(
+ session,
+ query_defuzzed,
+ query_limit=server.config.database.max_hits,
+ )
+ threads = plugins.messages.ThreadConstructor(results)
+ thread_struct, authors = await server.runners.run(threads.construct)
+ for (
+ thread
+ ) in (
+ thread_struct
+ ): # Make a list item for each thread (not for each email)
+ author = "Unknown"
+ date = "Unknown"
+ count = count_replies(thread)
+ # Find the email in the results pile and grab author and date
+ for k in results:
+ if k["id"] == thread["tid"]:
+ author = html.escape(k["from"])
+ date = html.escape(k["date"])
+ break
+ output += f"""- <a href="?thread={thread["tid"]}">{thread["subject"]}</a> - posted by {author} on {date} UTC, {count} replies.<br/>\n"""
+ # No month specified, which means just show all months with email in 'em
+ else:
+ title = list_id
+ canonical_link = f"/list.html?{list_id}"
+ output = f"""<link rel="canonical" href="/list.html?{list_id}" />\n"""
+ query_defuzzed_nodate = plugins.defuzzer.defuzz(mydata, nodate=True)
+ (
+ oldest,
+ youngest,
+ active_months,
+ ) = await plugins.messages.get_activity_span(
+ session, query_defuzzed_nodate
+ )
+ for month, activity in active_months.items():
+ output += (
+ f"""<a href="?list={list_id}&date={month}">{month}</a><br/>"""
+ )
+ else: # Just list all lists?
+ canonical_link = "/"
+ output = f"""<link rel="canonical" href="/" />\n"""
+ # Sort by domain, then by list name
+ for ml in sorted(server.data.lists.keys(), key=lambda x: x.split("@", 1)[-1] + "-" + x.split("@", 1)[0]):
+ entry = server.data.lists[ml]
+ if "@" in ml:
+ if not entry.get("private", True): # Only index public lists
+ output += f"<a href='?list={ml}'>{ml}</a><br/>\n"
+
+ if output and canonical_link:
+ output_interpolated = f"""
+ <html>
+ <head>
+ <link rel="canonical" href="{canonical_link}" />
+ <title>{title}</title>
+ </head>
+ <body>
+ <i>You are viewing a plain text version of this content. The canonical link for it is <a href="{canonical_link}">here</a>.</i><hr/>
+ {output}
+ </body>
+ </html>
+ """
+ return aiohttp.web.Response(
+ headers={"Content-Type": "text/html; charset=utf-8"},
+ status=200,
+ text=output_interpolated,
+ )
+ else:
+ return aiohttp.web.Response(
+ headers={"Content-Type": "text/plain"},
+ status=200,
+ text="No data",
+ )
+
+
+def register(server: plugins.server.BaseServer):
+ return plugins.server.Endpoint(process)
[incubator-ponymail-foal] 02/02: bump version
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 71754e1145962f1f3bbd22ef4a2abbce13d2b2d0
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Mon Jan 24 23:52:59 2022 +0100
bump version
---
server/server_version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/server_version.py b/server/server_version.py
index 699f4b6..562099f 100644
--- a/server/server_version.py
+++ b/server/server_version.py
@@ -1,2 +1,2 @@
# This file is generated by server/update_version.sh
-PONYMAIL_SERVER_VERSION = '35f3003'
+PONYMAIL_SERVER_VERSION = 'bbd540b'