You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/08/25 20:27:25 UTC
[incubator-ponymail-foal] branch master updated (8a1baf1 -> 7ece67f)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from 8a1baf1 store sources as sha3-256 of themselves, add a permalink reference to the digested doc.
new e1520ea Yank out older generators for new setups.
new 0a1fd77 remove unused imports
new cf6d870 this is now a prefix, not a index name
new 6c09deb this is .yaml now
new dece1db reword, plural
new 2c93f74 validate cli generator argument
new ef540a2 switch to having defaults
new e596124 Move mapping setup out of the .py file
new 7ece67f rename index creator, move things around a bit
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tools/mappings.yaml | 126 ++++++++++++++++++++++++
tools/setup.py | 272 +++++++++++++++++++---------------------------------
2 files changed, 222 insertions(+), 176 deletions(-)
create mode 100644 tools/mappings.yaml
[incubator-ponymail-foal] 04/09: this is .yaml now
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 6c09debb09617117476bab934c10ec696f911156
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:46:22 2020 +0200
this is .yaml now
---
tools/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/setup.py b/tools/setup.py
index 373d902..844b0c7 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -66,7 +66,7 @@ parser.add_argument(
"--clobber",
dest="clobber",
action="store_true",
- help="Allow overwrite of ponymail.cfg & ../site/api/lib/config.lua (default: create *.tmp if either exists)",
+ help="Allow overwrite of ponymail.yaml & ../site/api/lib/config.lua (default: create *.tmp if either exists)",
)
parser.add_argument("--dbhost", dest="dbhost", type=str, help="ES backend hostname")
parser.add_argument("--dbport", dest="dbport", type=str, help="DB port")
[incubator-ponymail-foal] 07/09: switch to having defaults
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit ef540a2f788db3715ef93a68c713d68cd83ed941
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 22:13:58 2020 +0200
switch to having defaults
This should speed up things for new users, by falling back to sane
defaults instead of demanding input.
---
tools/setup.py | 63 +++++++++++++++++++++++++++++++++++++---------------------
1 file changed, 40 insertions(+), 23 deletions(-)
diff --git a/tools/setup.py b/tools/setup.py
index 99841c1..1c6182b 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -172,35 +172,43 @@ if args.generator:
sys.stderr.write("Invalid generator specified. Must be one of: " + ", ".join(supported_generators) + "\n")
sys.exit(-1)
-while hostname == "":
- hostname = input(
- "What is the hostname of the ElasticSearch server? (e.g. localhost): "
- )
+hostname = input(
+ "What is the hostname of the ElasticSearch server? [localhost]: "
+)
+if not hostname:
+ hostname = "localhost"
-while urlPrefix == None:
- urlPrefix = input("Database URL prefix if any (hit enter if none): ")
+urlPrefix = input("Database URL prefix if any (hit enter if none): ")
-while port < 1:
+while port < 1 or port > 65536:
try:
- port = int(input("What port is ElasticSearch listening on? (normally 9200): "))
+ port = input("What port is ElasticSearch listening on? [9200]: ")
+ if not port:
+ port = 9200
+ port = int(port)
except ValueError:
pass
-while dbname == "":
- dbname = input("What would you like to call the mail index (e.g. ponymail): ")
+dbname = input("What would you like to call the mail index [ponymail]: ")
+if not dbname:
+ dbname = "ponymail"
-while mlserver == "":
- mlserver = input(
- "What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): "
- )
+mlserver = input(
+ "What is the hostname of the outgoing mailserver hostname? [localhost]: "
+)
+if not mlserver:
+ mlserver = "localhost"
-while mldom == "":
- mldom = input(
- "Which domains would you accept mail to from web-replies? (e.g. foo.org or *): "
- )
+mldom = input(
+ "Which domains would you accept mail to from web-replies? [*]: "
+)
+if not mldom:
+ mldom = "*"
-while wc == "":
- wc = input("Would you like to enable the word cloud feature? (Y/N): ")
+while wc.lower() not in ["y", "n"]:
+ wc = input("Would you like to enable the word cloud feature? (Y/N) [Y]: ").lower()
+ if not wc:
+ wc = "y"
if wc.lower() == "y":
wce = True
@@ -213,7 +221,10 @@ while genname == "":
"2 FULL: Full message digest with MTA trail. Not recommended for clustered setups."
)
try:
- gno = int(input("Please select a generator [1-2]: "))
+ gno = input("Please select a generator (1 or 2) [1]: ")
+ if not gno:
+ gno = 1
+ gno = int(gno)
if gno <= len(supported_generators) and supported_generators[gno - 1]:
genname = supported_generators[gno - 1]
except ValueError:
@@ -233,13 +244,19 @@ if genname == "dkim":
while shards < 1:
try:
- shards = int(input("How many shards for the ElasticSearch index? "))
+ shards = input("How many shards for the ElasticSearch index? [3]: ")
+ if not shards:
+ shards = 3
+ shards = int(shards)
except ValueError:
pass
while replicas < 0:
try:
- replicas = int(input("How many replicas for each shard? "))
+ replicas = input("How many replicas for each shard? [1]: ")
+ if not replicas:
+ replicas = 1
+ replicas = int(replicas)
except ValueError:
pass
[incubator-ponymail-foal] 06/09: validate cli generator argument
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 2c93f74ba31a71e4827e62a4648782b366c32083
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:49:50 2020 +0200
validate cli generator argument
move supported generators up, so we can validate against it if
the user specifies it on the command line.
---
tools/setup.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/tools/setup.py b/tools/setup.py
index 3b3b186..99841c1 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -127,6 +127,7 @@ shards = 0
replicas = -1
urlPrefix = None
nonce = None
+supported_generators = ["dkim", "full"]
# If called with --defaults (like from Docker), use default values
if args.defaults:
@@ -165,7 +166,11 @@ if args.dbshards:
if args.dbreplicas:
replicas = args.dbreplicas
if args.generator:
- genname = args.generator
+ if args.generator in supported_generators:
+ genname = args.generator
+ else:
+ sys.stderr.write("Invalid generator specified. Must be one of: " + ", ".join(supported_generators) + "\n")
+ sys.exit(-1)
while hostname == "":
hostname = input(
@@ -200,7 +205,6 @@ while wc == "":
wce = True
while genname == "":
- gens = ["dkim", "full"]
print("Please select a document ID generator:")
print(
"1 [RECOMMENDED] DKIM/RFC-6376: Short SHA3 hash useful for cluster setups with permalink usage"
@@ -210,8 +214,8 @@ while genname == "":
)
try:
gno = int(input("Please select a generator [1-2]: "))
- if gno <= len(gens) and gens[gno - 1]:
- genname = gens[gno - 1]
+ if gno <= len(supported_generators) and supported_generators[gno - 1]:
+ genname = supported_generators[gno - 1]
except ValueError:
pass
[incubator-ponymail-foal] 03/09: this is now a prefix,
not a index name
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit cf6d870b3bbd0a243dc7d367c68c5c246ba4ed12
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:45:52 2020 +0200
this is now a prefix, not a index name
---
tools/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/setup.py b/tools/setup.py
index 0afd10d..373d902 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -70,7 +70,7 @@ parser.add_argument(
)
parser.add_argument("--dbhost", dest="dbhost", type=str, help="ES backend hostname")
parser.add_argument("--dbport", dest="dbport", type=str, help="DB port")
-parser.add_argument("--dbname", dest="dbname", type=str, help="ES DB name")
+parser.add_argument("--dbname", dest="dbname", type=str, help="ES DB prefix")
parser.add_argument("--dbshards", dest="dbshards", type=int, help="DB Shard Count")
parser.add_argument(
"--dbreplicas", dest="dbreplicas", type=int, help="DB Replica Count"
[incubator-ponymail-foal] 09/09: rename index creator,
move things around a bit
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 7ece67fadcf5acd21b131637d5eeb16b05b474ec
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 22:25:47 2020 +0200
rename index creator, move things around a bit
---
tools/setup.py | 84 ++++++++++++++++++++++++++++++----------------------------
1 file changed, 43 insertions(+), 41 deletions(-)
diff --git a/tools/setup.py b/tools/setup.py
index 9b07016..700a644 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -22,6 +22,48 @@ import shutil
import sys
import yaml
+
+hostname = ""
+port = 0
+dbname = ""
+mlserver = ""
+mldom = ""
+wc = ""
+genname = ""
+wce = False
+shards = 0
+replicas = -1
+urlPrefix = None
+nonce = None
+supported_generators = ["dkim", "full"]
+
+
+def create_indices():
+ """Creates new indices for a fresh pony mail installation, it possible"""
+ # Check if index already exists
+ if es.indices.exists(dbname + "-mbox"):
+ if args.soe:
+ print(
+ "ElasticSearch indices with prefix '%s' already exists and SOE set, exiting quietly"
+ % dbname
+ )
+ sys.exit(0)
+ else:
+ print("Error: Existing ElasticSearch indices with prefix '%s' already exist!" % dbname)
+ sys.exit(-1)
+
+ print(f"Creating indices {dbname}-*...")
+
+ settings = {"number_of_shards": shards, "number_of_replicas": replicas}
+ mappings = yaml.safe_load(open("mappings.yaml", "r"))
+ for index, mappings in mappings.items():
+ res = es.indices.create(
+ index=f"{dbname}-{index}", body={"mappings": mappings, "settings": settings}
+ )
+
+ print(f"Index {dbname}-{index} created! %s " % res)
+
+
if sys.version_info <= (3, 7):
print("This script requires Python 3.8 or higher")
sys.exit(-1)
@@ -116,20 +158,6 @@ print("Let's start by determining some settings...")
print("")
-hostname = ""
-port = 0
-dbname = ""
-mlserver = ""
-mldom = ""
-wc = ""
-genname = ""
-wce = False
-shards = 0
-replicas = -1
-urlPrefix = None
-nonce = None
-supported_generators = ["dkim", "full"]
-
# If called with --defaults (like from Docker), use default values
if args.defaults:
hostname = "localhost"
@@ -263,32 +291,6 @@ while replicas < 0:
print("Okay, I got all I need, setting up Pony Mail...")
-
-def createIndex():
- # Check if index already exists
- if es.indices.exists(dbname + "-mbox"):
- if args.soe:
- print(
- "ElasticSearch indices with prefix '%s' already exists and SOE set, exiting quietly"
- % dbname
- )
- sys.exit(0)
- else:
- print("Error: Existing ElasticSearch indices with prefix '%s' already exist!" % dbname)
- sys.exit(-1)
-
- print(f"Creating indices {dbname}-*...")
-
- settings = {"number_of_shards": shards, "number_of_replicas": replicas}
- mappings = yaml.safe_load(open("mappings.yaml", "r"))
- for index, mappings in mappings.items():
- res = es.indices.create(
- index=f"{dbname}-{index}", body={"mappings": mappings, "settings": settings}
- )
-
- print(f"Index {dbname}-{index} created! %s " % res)
-
-
# we need to connect to database to determine the engine version
es = Elasticsearch(
[{"host": hostname, "port": port, "use_ssl": False, "url_prefix": urlPrefix}],
@@ -330,7 +332,7 @@ if DB_MAJOR == 0: # not known
if not args.noi:
try:
- createIndex()
+ create_indices()
except ElasticsearchException as e:
print("Index creation failed: %s" % e)
sys.exit(1)
[incubator-ponymail-foal] 08/09: Move mapping setup out of the .py
file
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit e596124c7634d8037382369ce74704506fd84e32
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 22:18:25 2020 +0200
Move mapping setup out of the .py file
---
tools/mappings.yaml | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tools/setup.py | 100 +----------------------------------------
2 files changed, 128 insertions(+), 98 deletions(-)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
new file mode 100644
index 0000000..603fc96
--- /dev/null
+++ b/tools/mappings.yaml
@@ -0,0 +1,126 @@
+account:
+ properties:
+ cid:
+ type: keyword
+ credentials:
+ properties:
+ altemail:
+ type: object
+ email:
+ type: keyword
+ fullname:
+ type: keyword
+ uid:
+ type: keyword
+ internal:
+ properties:
+ cookie:
+ type: keyword
+ ip:
+ type: keyword
+ oauth_used:
+ type: keyword
+ request_id:
+ type: keyword
+attachment:
+ properties:
+ source:
+ type: binary
+mailinglist:
+ properties:
+ description:
+ type: keyword
+ list:
+ type: keyword
+ name:
+ type: keyword
+mbox:
+ properties:
+ '@import_timestamp':
+ format: yyyy/MM/dd HH:mm:ss||yyyy/MM/dd
+ type: date
+ attachments:
+ properties:
+ content_type:
+ type: keyword
+ filename:
+ type: keyword
+ hash:
+ type: keyword
+ size:
+ type: long
+ body:
+ type: text
+ cc:
+ type: text
+ date:
+ format: yyyy/MM/dd HH:mm:ss
+ store: true
+ type: date
+ epoch:
+ type: long
+ from:
+ type: text
+ from_raw:
+ type: keyword
+ in-reply-to:
+ type: keyword
+ list:
+ type: text
+ list_raw:
+ type: keyword
+ message-id:
+ type: keyword
+ mid:
+ type: keyword
+ permalink:
+ type: keyword
+ private:
+ type: boolean
+ references:
+ type: text
+ subject:
+ fielddata: true
+ type: text
+ to:
+ type: text
+notification:
+ properties:
+ date:
+ format: yyyy/MM/dd HH:mm:ss
+ store: true
+ type: date
+ epoch:
+ type: long
+ from:
+ type: text
+ in-reply-to:
+ type: keyword
+ list:
+ type: text
+ message-id:
+ type: keyword
+ mid:
+ type: text
+ private:
+ type: boolean
+ recipient:
+ type: keyword
+ seen:
+ type: long
+ subject:
+ type: keyword
+ to:
+ type: text
+ type:
+ type: keyword
+source:
+ properties:
+ message-id:
+ type: keyword
+ mid:
+ type: keyword
+ permalink:
+ type: keyword
+ source:
+ type: binary
diff --git a/tools/setup.py b/tools/setup.py
index 1c6182b..9b07016 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -20,6 +20,7 @@ import logging
import os.path
import shutil
import sys
+import yaml
if sys.version_info <= (3, 7):
print("This script requires Python 3.8 or higher")
@@ -279,104 +280,7 @@ def createIndex():
print(f"Creating indices {dbname}-*...")
settings = {"number_of_shards": shards, "number_of_replicas": replicas}
-
- mappings = {
- "mbox": {
- "properties": {
- "@import_timestamp": {
- "type": "date",
- "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd",
- },
- "attachments": {
- "properties": {
- "content_type": {"type": "keyword",},
- "filename": {"type": "keyword",},
- "hash": {"type": "keyword",},
- "size": {"type": "long"},
- }
- },
- "body": {"type": "text"},
- "cc": {"type": "text"},
- "date": {
- "type": "date",
- "store": True,
- "format": "yyyy/MM/dd HH:mm:ss",
- },
- "epoch": {"type": "long",}, # number of seconds since the epoch
- "from": {"type": "text"},
- "from_raw": {"type": "keyword",},
- "in-reply-to": {"type": "keyword",},
- "list": {"type": "text"},
- "list_raw": {"type": "keyword",},
- "message-id": {"type": "keyword",},
- "mid": {"type": "keyword"},
- "private": {"type": "boolean"},
- "permalink": {"type": "keyword"},
- "references": {"type": "text"},
- "subject": {"type": "text", "fielddata": True},
- "to": {"type": "text"},
- }
- },
- "attachment": {"properties": {"source": {"type": "binary"}}},
- "source": {
- "properties": {
- "source": {"type": "binary"},
- "message-id": {"type": "keyword",},
- "permalink": {"type": "keyword"},
- "mid": {"type": "keyword"},
- }
- },
- "mailinglist": {
- "properties": {
- "description": {"type": "keyword",},
- "list": {"type": "keyword",},
- "name": {"type": "keyword",},
- }
- },
- "account": {
- "properties": {
- "cid": {"type": "keyword",},
- "credentials": {
- "properties": {
- "altemail": {"type": "object"},
- "email": {"type": "keyword",},
- "fullname": {"type": "keyword",},
- "uid": {"type": "keyword",},
- }
- },
- "internal": {
- "properties": {
- "cookie": {"type": "keyword",},
- "ip": {"type": "keyword",},
- "oauth_used": {"type": "keyword",},
- }
- },
- "request_id": {"type": "keyword",},
- }
- },
- "notification": {
- "properties": {
- "date": {
- "type": "date",
- "store": True,
- "format": "yyyy/MM/dd HH:mm:ss",
- },
- "epoch": {"type": "long"},
- "from": {"type": "text",},
- "in-reply-to": {"type": "keyword",},
- "list": {"type": "text",},
- "message-id": {"type": "keyword",},
- "mid": {"type": "text",},
- "private": {"type": "boolean"},
- "recipient": {"type": "keyword",},
- "seen": {"type": "long"},
- "subject": {"type": "keyword",},
- "to": {"type": "text",},
- "type": {"type": "keyword",},
- }
- },
- }
-
+ mappings = yaml.safe_load(open("mappings.yaml", "r"))
for index, mappings in mappings.items():
res = es.indices.create(
index=f"{dbname}-{index}", body={"mappings": mappings, "settings": settings}
[incubator-ponymail-foal] 02/09: remove unused imports
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 0a1fd77455f7daa09a3450068ef91d9ca5be9186
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:45:34 2020 +0200
remove unused imports
---
tools/setup.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/tools/setup.py b/tools/setup.py
index 546ed01..0afd10d 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -17,11 +17,9 @@
import argparse
import importlib.util
import logging
-import os
import os.path
import shutil
import sys
-import yaml
if sys.version_info <= (3, 7):
print("This script requires Python 3.8 or higher")
[incubator-ponymail-foal] 05/09: reword, plural
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit dece1db7e6485fb59f7f8c6718568362080a9335
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:47:02 2020 +0200
reword, plural
---
tools/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/setup.py b/tools/setup.py
index 844b0c7..3b3b186 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -97,7 +97,7 @@ parser.add_argument(
"--noindex",
dest="noi",
action="store_true",
- help="Don't make an ES index, assume it exists",
+ help="Don't create ElasticSearch indices, assume they exist",
)
parser.add_argument(
"--nocloud", dest="nwc", action="store_true", help="Do not enable word cloud"
[incubator-ponymail-foal] 01/09: Yank out older generators for new
setups.
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit e1520ea68eaf0145131169f66ae1e34f02e3033b
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 21:45:17 2020 +0200
Yank out older generators for new setups.
We should be supporting the 'full' and 'dkim' generators only, as they
are sufficiently unique. the dkim generator is likely to change, pending
PR review, but that's fine.
---
tools/setup.py | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/tools/setup.py b/tools/setup.py
index 3eba8cd..546ed01 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -202,21 +202,16 @@ while wc == "":
wce = True
while genname == "":
- gens = ["legacy", "medium", "cluster", "full", "dkim"]
+ gens = ["dkim", "full"]
print("Please select a document ID generator:")
print(
- "1 LEGACY: The original document generator for v/0.1-0.8 (no longer recommended)"
+ "1 [RECOMMENDED] DKIM/RFC-6376: Short SHA3 hash useful for cluster setups with permalink usage"
)
print(
- "2 MEDIUM: The medium comprehensive generator for v/0.9 (no longer recommended)"
- )
- print("3 REDUNDANT: Near-full message digest, discard MTA trail")
- print("4 FULL: Full message digest with MTA trail")
- print(
- "5 [RECOMMENDED] DKIM/RFC-6376: Short SHA3 hash useful for cluster setups with permalink usage"
+ "2 FULL: Full message digest with MTA trail. Not recommended for clustered setups."
)
try:
- gno = int(input("Please select a generator [1-5]: "))
+ gno = int(input("Please select a generator [1-2]: "))
if gno <= len(gens) and gens[gno - 1]:
genname = gens[gno - 1]
except ValueError:
@@ -459,7 +454,7 @@ elasticsearch:
#backup: database name
archiver:
- #generator: medium|full|cluster|dkim|other (dkim recommended)
+ #generator: dkim|full (dkim recommended)
generator: %s
nonce: %s