[incubator-ponymail] branch master updated: Bug: setup.py uses ES library version

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git


The following commit(s) were added to refs/heads/master by this push:
     new 1c2a807  Bug: setup.py uses ES library version
1c2a807 is described below

commit 1c2a807098d95fdd133a8e9bf015678baf7d0d3f
Author: Sebb <se...@apache.org>
AuthorDate: Sun May 27 12:37:29 2018 +0100

    Bug: setup.py uses ES library version
    
    to decide what features the database supports
    This fixes #464
---
 CHANGELOG.md   |  1 +
 tools/setup.py | 70 +++++++++++++++++++++++++++++++++++++++-------------------
 2 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 13d6768..6f0fd58 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## Changes in 0.11-SNAPSHOT
+- Bug: setup.py uses ES library version to decide what features the database supports (#464)
 - Various tidyups suggested by Pylint
 - Bug: archiver.py can never detect content-type: flowed (#461)
 - Bug: import-mbox.py: imap code should not reset ES instance (#460)
diff --git a/tools/setup.py b/tools/setup.py
index d5825bb..1463f2b 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -19,6 +19,7 @@ import getpass
 import subprocess
 import argparse
 import shutil
+import logging
 
 if sys.version_info <= (3, 3):
     print("This script requires Python 3.4 or higher")
@@ -27,8 +28,6 @@ if sys.version_info <= (3, 3):
 dopip = False
 try:
     from elasticsearch import Elasticsearch
-    from elasticsearch import VERSION as ES_VERSION
-    ES_MAJOR = ES_VERSION[0]
 except ImportError:
     dopip = True
     
@@ -43,6 +42,7 @@ elif dopip:
     print("Hang on!")
     try:
         subprocess.check_call(('pip3','install','elasticsearch','formatflowed', 'netaddr', 'certifi'))
+        # retry the import
         from elasticsearch import Elasticsearch
     except ImportError:
         print("Oh dear, looks like this failed :(")
@@ -50,6 +50,11 @@ elif dopip:
         print("pip install elasticsearch formatflowed netaddr certifi")
         sys.exit(-1)
 
+# at this point we can assume elasticsearch is loaded
+from elasticsearch import ElasticsearchException
+from elasticsearch import ConnectionError as ES_ConnectionError
+from elasticsearch import VERSION as ES_VERSION
+ES_MAJOR = ES_VERSION[0]
 
 # CLI arg parsing
 parser = argparse.ArgumentParser(description='Command line options.')
@@ -195,21 +200,6 @@ while replicas < 0:
 print("Okay, I got all I need, setting up Pony Mail...")
 
 def createIndex():
-    es = Elasticsearch([
-        {
-            'host': hostname,
-            'port': port,
-            'use_ssl': False,
-            'url_prefix': urlPrefix
-        }],
-        max_retries=5,
-        retry_on_timeout=True
-        )
-
-    DB_VERSION=es.info()['version']['number']
-    DB_MAJOR=int(DB_VERSION.split('.')[0])
-    print("Versions: library %d (%s), engine %d (%s)" % (ES_MAJOR, '.'.join(map(str,ES_VERSION)) , DB_MAJOR, DB_VERSION))
-
     # Check if index already exists
     if es.indices.exists(dbname):
         if args.soe:
@@ -472,14 +462,48 @@ def createIndex():
         )
     
     print("Index created! %s " % res)
-    
+
+# we need to connect to database to determine the engine version   
+es = Elasticsearch([
+    {
+        'host': hostname,
+        'port': port,
+        'use_ssl': False,
+        'url_prefix': urlPrefix
+    }],
+    max_retries=5,
+    retry_on_timeout=True
+    )
+
+# elasticsearch logs lots of warnings on retries/connection failure
+logging.getLogger("elasticsearch").setLevel(logging.ERROR)
+
+try:
+    DB_VERSION=es.info()['version']['number']
+except ES_ConnectionError:
+    print("WARNING: Connection error: could not determine the engine version.")
+    DB_VERSION='0.0.0'
+
+DB_MAJOR=int(DB_VERSION.split('.')[0])
+print("Versions: library %d (%s), engine %d (%s)" % (ES_MAJOR, '.'.join(map(str,ES_VERSION)) , DB_MAJOR, DB_VERSION))
+
+if not DB_MAJOR == ES_MAJOR:
+    print("WARNING: library version does not agree with engine version!")
+
+if DB_MAJOR == 0: # not known
+    if args.noi:
+        # allow setup to be used without engine running
+        print("Could not determine the engine version. Assume it is the same as the library version.")
+        DB_MAJOR = ES_MAJOR
+    else:
+        # if we cannot connect to get the version, we cannot create the index later
+        print("Could not connect to the engine. Fatal.")
+        sys.exit(1)
+
 if not args.noi:
     try:
-        import logging
-        # elasticsearch logs lots of warnings on retries/connection failure
-        logging.getLogger("elasticsearch").setLevel(logging.ERROR)
         createIndex()
-    except Exception as e:
+    except ElasticsearchException as e:
         print("Index creation failed: %s" % e)
         sys.exit(1)
 
@@ -519,7 +543,7 @@ generator:              %s
 
 ###############################################################
             """ % (hostname, dbname, port, 
-                   'wait:                  active shard count' if ES_MAJOR == 5 else 'write:                 consistency level (default quorum)', genname))
+                   'wait:                  active shard count' if DB_MAJOR == 5 else 'write:                 consistency level (default quorum)', genname))
 
 config_path = "../site/api/lib"
 config_file = "config.lua"

-- 
To stop receiving notification emails like this one, please contact
sebb@apache.org.