You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/08/25 20:27:33 UTC
[incubator-ponymail-foal] 08/09: Move mapping setup out of the .py
file
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit e596124c7634d8037382369ce74704506fd84e32
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 25 22:18:25 2020 +0200
Move mapping setup out of the .py file
---
tools/mappings.yaml | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tools/setup.py | 100 +----------------------------------------
2 files changed, 128 insertions(+), 98 deletions(-)
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
new file mode 100644
index 0000000..603fc96
--- /dev/null
+++ b/tools/mappings.yaml
@@ -0,0 +1,126 @@
+account:
+ properties:
+ cid:
+ type: keyword
+ credentials:
+ properties:
+ altemail:
+ type: object
+ email:
+ type: keyword
+ fullname:
+ type: keyword
+ uid:
+ type: keyword
+ internal:
+ properties:
+ cookie:
+ type: keyword
+ ip:
+ type: keyword
+ oauth_used:
+ type: keyword
+ request_id:
+ type: keyword
+attachment:
+ properties:
+ source:
+ type: binary
+mailinglist:
+ properties:
+ description:
+ type: keyword
+ list:
+ type: keyword
+ name:
+ type: keyword
+mbox:
+ properties:
+ '@import_timestamp':
+ format: yyyy/MM/dd HH:mm:ss||yyyy/MM/dd
+ type: date
+ attachments:
+ properties:
+ content_type:
+ type: keyword
+ filename:
+ type: keyword
+ hash:
+ type: keyword
+ size:
+ type: long
+ body:
+ type: text
+ cc:
+ type: text
+ date:
+ format: yyyy/MM/dd HH:mm:ss
+ store: true
+ type: date
+ epoch:
+ type: long
+ from:
+ type: text
+ from_raw:
+ type: keyword
+ in-reply-to:
+ type: keyword
+ list:
+ type: text
+ list_raw:
+ type: keyword
+ message-id:
+ type: keyword
+ mid:
+ type: keyword
+ permalink:
+ type: keyword
+ private:
+ type: boolean
+ references:
+ type: text
+ subject:
+ fielddata: true
+ type: text
+ to:
+ type: text
+notification:
+ properties:
+ date:
+ format: yyyy/MM/dd HH:mm:ss
+ store: true
+ type: date
+ epoch:
+ type: long
+ from:
+ type: text
+ in-reply-to:
+ type: keyword
+ list:
+ type: text
+ message-id:
+ type: keyword
+ mid:
+ type: text
+ private:
+ type: boolean
+ recipient:
+ type: keyword
+ seen:
+ type: long
+ subject:
+ type: keyword
+ to:
+ type: text
+ type:
+ type: keyword
+source:
+ properties:
+ message-id:
+ type: keyword
+ mid:
+ type: keyword
+ permalink:
+ type: keyword
+ source:
+ type: binary
diff --git a/tools/setup.py b/tools/setup.py
index 1c6182b..9b07016 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -20,6 +20,7 @@ import logging
import os.path
import shutil
import sys
+import yaml
if sys.version_info <= (3, 7):
print("This script requires Python 3.8 or higher")
@@ -279,104 +280,7 @@ def createIndex():
print(f"Creating indices {dbname}-*...")
settings = {"number_of_shards": shards, "number_of_replicas": replicas}
-
- mappings = {
- "mbox": {
- "properties": {
- "@import_timestamp": {
- "type": "date",
- "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd",
- },
- "attachments": {
- "properties": {
- "content_type": {"type": "keyword",},
- "filename": {"type": "keyword",},
- "hash": {"type": "keyword",},
- "size": {"type": "long"},
- }
- },
- "body": {"type": "text"},
- "cc": {"type": "text"},
- "date": {
- "type": "date",
- "store": True,
- "format": "yyyy/MM/dd HH:mm:ss",
- },
- "epoch": {"type": "long",}, # number of seconds since the epoch
- "from": {"type": "text"},
- "from_raw": {"type": "keyword",},
- "in-reply-to": {"type": "keyword",},
- "list": {"type": "text"},
- "list_raw": {"type": "keyword",},
- "message-id": {"type": "keyword",},
- "mid": {"type": "keyword"},
- "private": {"type": "boolean"},
- "permalink": {"type": "keyword"},
- "references": {"type": "text"},
- "subject": {"type": "text", "fielddata": True},
- "to": {"type": "text"},
- }
- },
- "attachment": {"properties": {"source": {"type": "binary"}}},
- "source": {
- "properties": {
- "source": {"type": "binary"},
- "message-id": {"type": "keyword",},
- "permalink": {"type": "keyword"},
- "mid": {"type": "keyword"},
- }
- },
- "mailinglist": {
- "properties": {
- "description": {"type": "keyword",},
- "list": {"type": "keyword",},
- "name": {"type": "keyword",},
- }
- },
- "account": {
- "properties": {
- "cid": {"type": "keyword",},
- "credentials": {
- "properties": {
- "altemail": {"type": "object"},
- "email": {"type": "keyword",},
- "fullname": {"type": "keyword",},
- "uid": {"type": "keyword",},
- }
- },
- "internal": {
- "properties": {
- "cookie": {"type": "keyword",},
- "ip": {"type": "keyword",},
- "oauth_used": {"type": "keyword",},
- }
- },
- "request_id": {"type": "keyword",},
- }
- },
- "notification": {
- "properties": {
- "date": {
- "type": "date",
- "store": True,
- "format": "yyyy/MM/dd HH:mm:ss",
- },
- "epoch": {"type": "long"},
- "from": {"type": "text",},
- "in-reply-to": {"type": "keyword",},
- "list": {"type": "text",},
- "message-id": {"type": "keyword",},
- "mid": {"type": "text",},
- "private": {"type": "boolean"},
- "recipient": {"type": "keyword",},
- "seen": {"type": "long"},
- "subject": {"type": "keyword",},
- "to": {"type": "text",},
- "type": {"type": "keyword",},
- }
- },
- }
-
+ mappings = yaml.safe_load(open("mappings.yaml", "r"))
for index, mappings in mappings.items():
res = es.indices.create(
index=f"{dbname}-{index}", body={"mappings": mappings, "settings": settings}