You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ponymail.apache.org by GitBox <gi...@apache.org> on 2018/06/03 10:42:35 UTC
[GitHub] dimidd closed pull request #467: setup: move mappings to a separate
file
dimidd closed pull request #467: setup: move mappings to a separate file
URL: https://github.com/apache/incubator-ponymail/pull/467
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/tools/mappings.json b/tools/mappings.json
new file mode 100644
index 0000000..bf405c1
--- /dev/null
+++ b/tools/mappings.json
@@ -0,0 +1,228 @@
+{
+ "account": {
+ "_all": {
+ "enabled": false
+ },
+ "properties": {
+ "cid": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "credentials": {
+ "properties": {
+ "altemail": {
+ "type": "object"
+ },
+ "email": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "fullname": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "uid": {
+ "index": "not_analyzed",
+ "type": "string"
+ }
+ }
+ },
+ "internal": {
+ "properties": {
+ "cookie": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "ip": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "oauth_used": {
+ "index": "not_analyzed",
+ "type": "string"
+ }
+ }
+ },
+ "request_id": {
+ "index": "not_analyzed",
+ "type": "string"
+ }
+ }
+ },
+ "attachment": {
+ "properties": {
+ "source": {
+ "type": "binary"
+ }
+ }
+ },
+ "mailinglists": {
+ "_all": {
+ "enabled": false
+ },
+ "properties": {
+ "description": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "list": {
+ "type": "string"
+ },
+ "name": {
+ "index": "not_analyzed",
+ "type": "string"
+ }
+ }
+ },
+ "mbox": {
+ "properties": {
+ "@import_timestamp": {
+ "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd",
+ "type": "date"
+ },
+ "attachments": {
+ "properties": {
+ "content_type": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "filename": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "hash": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "size": {
+ "type": "long"
+ }
+ }
+ },
+ "body": {
+ "type": "string"
+ },
+ "cc": {
+ "type": "string"
+ },
+ "date": {
+ "format": "yyyy/MM/dd HH:mm:ss",
+ "index": "not_analyzed",
+ "store": true,
+ "type": "date"
+ },
+ "epoch": {
+ "index": "not_analyzed",
+ "type": "long"
+ },
+ "from": {
+ "type": "string"
+ },
+ "from_raw": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "in-reply-to": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "list": {
+ "type": "string"
+ },
+ "list_raw": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "message-id": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "mid": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "references": {
+ "type": "string"
+ },
+ "subject": {
+ "fielddata": true,
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ }
+ }
+ },
+ "mbox_source": {
+ "_all": {
+ "enabled": false
+ },
+ "properties": {
+ "message-id": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "mid": {
+ "type": "string"
+ },
+ "source": {
+ "type": "binary"
+ }
+ }
+ },
+ "notifications": {
+ "_all": {
+ "enabled": false
+ },
+ "properties": {
+ "date": {
+ "format": "yyyy/MM/dd HH:mm:ss",
+ "store": true,
+ "type": "date"
+ },
+ "epoch": {
+ "type": "long"
+ },
+ "from": {
+ "type": "string"
+ },
+ "in-reply-to": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "list": {
+ "type": "string"
+ },
+ "message-id": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "mid": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "recipient": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "seen": {
+ "type": "long"
+ },
+ "subject": {
+ "fielddata": true,
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ },
+ "type": {
+ "index": "not_analyzed",
+ "type": "string"
+ }
+ }
+ }
+}
diff --git a/tools/setup.py b/tools/setup.py
index 1463f2b..c9abb01 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -20,6 +20,7 @@
import argparse
import shutil
import logging
+import json
if sys.version_info <= (3, 3):
print("This script requires Python 3.4 or higher")
@@ -30,7 +31,7 @@
from elasticsearch import Elasticsearch
except ImportError:
dopip = True
-
+
if dopip and (getpass.getuser() != "root"):
print("It looks like you need to install some python modules first")
print("Either run this as root to do so, or run: ")
@@ -59,7 +60,7 @@
# CLI arg parsing
parser = argparse.ArgumentParser(description='Command line options.')
-parser.add_argument('--defaults', dest='defaults', action='store_true',
+parser.add_argument('--defaults', dest='defaults', action='store_true',
help='Use default settings')
parser.add_argument('--dbprefix', dest='dbprefix')
parser.add_argument('--clobber', dest='clobber', action='store_true',
@@ -78,17 +79,17 @@
help='Host name of outgoing mail server')
parser.add_argument('--mldom', dest='mldom', type=str,
help='Domains to accept mail for via UI')
-parser.add_argument('--wordcloud', dest='wc', action='store_true',
+parser.add_argument('--wordcloud', dest='wc', action='store_true',
help='Enable word cloud')
-parser.add_argument('--skiponexist', dest='soe', action='store_true',
+parser.add_argument('--skiponexist', dest='soe', action='store_true',
help='Skip setup if ES index exists')
-parser.add_argument('--noindex', dest='noi', action='store_true',
+parser.add_argument('--noindex', dest='noi', action='store_true',
help="Don't make an ES index, assume it exists")
-parser.add_argument('--nocloud', dest='nwc', action='store_true',
+parser.add_argument('--nocloud', dest='nwc', action='store_true',
help='Do not enable word cloud')
parser.add_argument('--generator', dest='generator', type=str,
help='Document ID Generator to use (legacy, medium, cluster, full)')
-args = parser.parse_args()
+args = parser.parse_args()
print("Welcome to the Pony Mail setup script!")
print("Let's start by determining some settings...")
@@ -144,13 +145,13 @@
replicas = args.dbreplicas
if args.generator:
genname = args.generator
-
+
while hostname == "":
hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ")
while urlPrefix == None:
urlPrefix = input("Database URL prefix if any (hit enter if none): ")
-
+
while port < 1:
try:
port = int(input("What port is ElasticSearch listening on? (normally 9200): "))
@@ -162,7 +163,7 @@
while mlserver == "":
mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ")
-
+
while mldom == "":
mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ")
@@ -184,7 +185,7 @@
genname = gens[gno-1]
except ValueError:
pass
-
+
while shards < 1:
try:
shards = int(input("How many shards for the ElasticSearch index? "))
@@ -216,240 +217,8 @@ def createIndex():
"number_of_replicas" : replicas
}
- mappings = {
- "mbox" : {
- "properties" : {
- "@import_timestamp" : {
- "type" : "date",
- "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
- },
- "attachments" : {
- "properties" : {
- "content_type" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "filename" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "hash" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "size" : {
- "type" : "long"
- }
- }
- },
- "body" : {
- "type" : "string"
- },
- "cc": {
- "type": "string"
- },
- "date" : {
- "type" : "date",
- "store" : True,
- "format" : "yyyy/MM/dd HH:mm:ss",
- "index" : "not_analyzed"
- },
- "epoch" : { # number of seconds since the epoch
- "type" : "long",
- "index" : "not_analyzed"
- },
- "from" : {
- "type" : "string"
- },
- "from_raw" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "in-reply-to" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "list" : {
- "type" : "string"
- },
- "list_raw" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "message-id" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "mid" : {
- "type" : "string"
- },
- "private" : {
- "type" : "boolean"
- },
- "references" : {
- "type" : "string"
- },
- "subject" : {
- "type" : "string",
- "fielddata": True # dropped later if DB_MAJOR==2
- },
- "to" : {
- "type" : "string"
- }
- }
- },
- "attachment" : {
- "properties" : {
- "source" : {
- "type" : "binary"
- }
- }
- },
- "mbox_source" : {
- "_all": {
- "enabled": False # this doc type is not searchable
- },
- "properties" : {
- "source" : {
- "type" : "binary"
- },
- "message-id" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "mid" : {
- "type" : "string"
- }
- }
- },
- "mailinglists" : {
- "_all": {
- "enabled": False # this doc type is not searchable
- },
- "properties" : {
- "description" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "list" : {
- "type" : "string",
-# "index" : "not_analyzed"
- },
- "name" : {
- "type" : "string",
- "index" : "not_analyzed"
- }
- }
- },
- "account" : {
- "_all": {
- "enabled": False # this doc type is not searchable
- },
- "properties" : {
- "cid" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "credentials" : {
- "properties" : {
- "altemail" : {
- "type" : "object"
- },
- "email" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "fullname" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "uid" : {
- "type" : "string",
- "index" : "not_analyzed"
- }
- }
- },
- "internal" : {
- "properties" : {
- "cookie" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "ip" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "oauth_used" : {
- "type" : "string",
- "index" : "not_analyzed"
- }
- }
- },
- "request_id" : {
- "type" : "string",
- "index" : "not_analyzed"
- }
- }
- },
- "notifications" : {
- "_all": {
- "enabled": False # this doc type is not searchable
- },
- "properties" : {
- "date" : {
- "type" : "date",
- "store" : True,
- "format" : "yyyy/MM/dd HH:mm:ss"
- },
- "epoch" : {
- "type" : "long"
- },
- "from" : {
- "type" : "string",
-# "index" : "not_analyzed"
- },
- "in-reply-to" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "list" : {
- "type" : "string",
-# "index" : "not_analyzed"
- },
- "message-id" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "mid" : {
- "type" : "string",
-# "index" : "not_analyzed"
- },
- "private" : {
- "type" : "boolean"
- },
- "recipient" : {
- "type" : "string",
- "index" : "not_analyzed"
- },
- "seen" : {
- "type" : "long"
- },
- "subject" : {
- "type" : "string",
- "fielddata": True # dropped later if DB_MAJOR==2
-# "index" : "not_analyzed"
- },
- "to" : {
- "type" : "string",
-# "index" : "not_analyzed"
- },
- "type" : {
- "type" : "string",
- "index" : "not_analyzed"
- }
- }
- }
- }
+ this_files_path = os.path.dirname(os.path.realpath(__file__))
+ mappings = json.load(open("%s/mappings.json" % this_files_path, "r"))
if DB_MAJOR == 2: # ES 2 handles fielddata differently
del mappings['mbox']['properties']['subject']['fielddata']
@@ -460,10 +229,10 @@ def createIndex():
"settings": settings
}
)
-
+
print("Index created! %s " % res)
-# we need to connect to database to determine the engine version
+# we need to connect to database to determine the engine version
es = Elasticsearch([
{
'host': hostname,
@@ -517,7 +286,7 @@ def createIndex():
with open(ponymail_cfg, "w") as f:
f.write("""
###############################################################
-# Pony Mail Configuration file
+# Pony Mail Configuration file
# Main ES configuration
[elasticsearch]
@@ -542,7 +311,7 @@ def createIndex():
#cropout: string to crop from list-id
###############################################################
- """ % (hostname, dbname, port,
+ """ % (hostname, dbname, port,
'wait: active shard count' if DB_MAJOR == 5 else 'write: consistency level (default quorum)', genname))
config_path = "../site/api/lib"
@@ -575,7 +344,7 @@ def createIndex():
-- env = 'subprocess' -- use environment vars instead of request headers
-- }
},
--- allow_insecure_cookie = true, -- override the default (false) - only use for test installations
+-- allow_insecure_cookie = true, -- override the default (false) - only use for test installations
-- no_association = {}, -- domains that are not allowed for email association
-- listsDisplay = 'regex', -- if defined, hide list names that don't match the regex
-- debug = false, -- whether to return debug information
@@ -583,11 +352,11 @@ def createIndex():
}
return config
""" % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
-
+
print("Copying sample JS config to config.js (if needed)...")
if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"):
shutil.copy("../site/js/config.js.sample", "../site/js/config.js")
-
-
+
+
print("All done, Pony Mail should...work now :)")
print("If you are using an external mail inbound server, \nmake sure to copy archiver.py and ponymail.cfg to it")
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services