You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by se...@apache.org on 2017/01/28 23:49:17 UTC
incubator-ponymail git commit: ES 5.0 no longer supports the write
consistency option; archiver fails
Repository: incubator-ponymail
Updated Branches:
refs/heads/master 06612e68a -> 848bb7129
ES 5.0 no longer supports the write consistency option; archiver fails
This fixes #351
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/848bb712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/848bb712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/848bb712
Branch: refs/heads/master
Commit: 848bb7129dede6f2c01ead4ce49ab070be6ad787
Parents: 06612e6
Author: Sebb <se...@apache.org>
Authored: Sat Jan 28 23:48:29 2017 +0000
Committer: Sebb <se...@apache.org>
Committed: Sat Jan 28 23:48:29 2017 +0000
----------------------------------------------------------------------
CHANGELOG.md | 1 +
tools/archiver.py | 31 +++++++++++++++++++++++++------
tools/setup.py | 7 +++++--
3 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e8d0d16..69ff74e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -134,6 +134,7 @@
- Use constant for max list count instead of 500000 (#352)
- Tighten wildcard searches to only search in the same domain level (#348)
- useless conditional when fetching id parameter (#353)
+- ES 5.0 no longer supports the write consistency option for index(); archiver fails (#351)
## CHANGES in 0.9b:
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index c03bf8f..bf4033b 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -40,6 +40,8 @@ sub someone to the list(s) and add this to their .forward file:
logger = None
from elasticsearch import Elasticsearch
+from elasticsearch import VERSION as ES_VERSION
+ES_MAJOR = ES_VERSION[0]
from formatflowed import convertToWrapped
import hashlib
import email.utils
@@ -140,6 +142,16 @@ class Archiver(object):
"x-mailman-rule-misses",
]
+ """ Intercept index calls and fix up consistency argument """
+ def index(self, **kwargs):
+ if ES_MAJOR == 5:
+ if kwargs.pop('consistency', None): # drop the key if present
+ if self.wait_for_active_shards: # replace with wait if defined
+ kwargs['wait_for_active_shards'] = self.wait_for_active_shards
+ return self.es.index(
+ **kwargs
+ )
+
def __init__(self, parseHTML=False):
""" Just initialize ES. """
self.html = parseHTML
@@ -148,7 +160,14 @@ class Archiver(object):
self.html2text = html2text.html2text
self.dbname = config.get("elasticsearch", "dbname")
ssl = config.get("elasticsearch", "ssl", fallback="false").lower() == 'true'
+ # Always allow this to be set; will be replaced as necessary by wait_for_active_shards
self.consistency = config.get('elasticsearch', 'write', fallback='quorum')
+ if ES_MAJOR == 2:
+ pass
+ elif ES_MAJOR == 5:
+ self.wait_for_active_shards = config.get('elasticsearch', 'wait', fallback=1)
+ else:
+ raise Exception("Unexpected elasticsearch version ", ES_VERSION)
self.cropout = config.get("debug", "cropout", fallback=None)
uri = config.get("elasticsearch", "uri", fallback="")
dbs = [
@@ -383,7 +402,7 @@ class Archiver(object):
if contents:
for key in contents:
- self.es.index(
+ self.index(
index=self.dbname,
doc_type="attachment",
id=key,
@@ -392,7 +411,7 @@ class Archiver(object):
}
)
- self.es.index(
+ self.index(
index=self.dbname,
doc_type="mbox",
id=ojson['mid'],
@@ -400,7 +419,7 @@ class Archiver(object):
body = ojson
)
- self.es.index(
+ self.index(
index=self.dbname,
doc_type="mbox_source",
id=ojson['mid'],
@@ -413,7 +432,7 @@ class Archiver(object):
# If MailMan and list info is present, save/update it in ES:
if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description and mlist.list_name:
- self.es.index(
+ self.index(
index=self.dbname,
doc_type="mailinglists",
id=lid,
@@ -441,7 +460,7 @@ class Archiver(object):
if doc:
oldrefs.append(cid)
# N.B. no index is supplied, so ES will generate one
- self.es.index(
+ self.index(
index=self.dbname,
doc_type="notifications",
consistency = self.consistency,
@@ -476,7 +495,7 @@ class Archiver(object):
if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications') == 'indirect' and not cid in oldrefs:
oldrefs.append(cid)
# N.B. no index is supplied, so ES will generate one
- self.es.index(
+ self.index(
index=self.dbname,
consistency = self.consistency,
doc_type="notifications",
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/tools/setup.py
----------------------------------------------------------------------
diff --git a/tools/setup.py b/tools/setup.py
index 65957af..1187286 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -27,6 +27,8 @@ if sys.version_info <= (3, 3):
dopip = False
try:
from elasticsearch import Elasticsearch
+ from elasticsearch import VERSION as ES_VERSION
+ ES_MAJOR = ES_VERSION[0]
except:
dopip = True
@@ -468,7 +470,7 @@ ssl: false
#user: username
#password: password
-#write: consistency level (default quorum)
+#%s
#backup: database name
@@ -479,7 +481,8 @@ ssl: false
#cropout: string to crop from list-id
###############################################################
- """ % (hostname, dbname, port))
+ """ % (hostname, dbname, port,
+ 'wait: active shard count' if ES_MAJOR == 5 else 'write: consistency level (default quorum)'))
f.close()
config_path = "../site/api/lib"