You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/03/31 09:48:22 UTC
[14/16] lens git commit: LENS-1398: Python client not raising
exception in some cases
LENS-1398: Python client not raising exception in some cases
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a3045fee
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a3045fee
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a3045fee
Branch: refs/heads/lens-1381
Commit: a3045fee12b3e750b6c931f5221a480d415efc76
Parents: bd344c7
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Mar 21 16:09:05 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Mar 21 16:09:05 2017 +0530
----------------------------------------------------------------------
contrib/clients/python/lens/client/main.py | 4 +-
contrib/clients/python/lens/client/query.py | 43 ++++++++++++++++-----
contrib/clients/python/setup.py | 2 +
contrib/clients/python/test/test_lensclient.py | 13 +++++--
tools/conf/server/lens-site.xml | 6 +++
5 files changed, 54 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/main.py
----------------------------------------------------------------------
diff --git a/contrib/clients/python/lens/client/main.py b/contrib/clients/python/lens/client/main.py
index bf5d81e..1e9d88f 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -21,7 +21,8 @@ from .log import LensLogClient
from .session import LensSessionClient
from .query import LensQueryClient
from .utils import xml_file_to_conf
-
+import logging
+logger = logging.getLogger(__name__)
class LensClient(object):
def __init__(self, base_url=None, username="", password="", database=None, conf=None):
@@ -35,6 +36,7 @@ class LensClient(object):
self.base_url = base_url or conf.get('lens.server.base.url', "http://0.0.0.0:9999/lensapi")
if self.base_url[-1] != '/':
self.base_url += "/"
+ logger.debug("Using conf %s", conf)
username = username or conf.get('lens.client.user.name', "anonymous")
database = database or conf.get('lens.client.dbname')
self.session = LensSessionClient(self.base_url, username, password, database, conf)
http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/query.py
----------------------------------------------------------------------
diff --git a/contrib/clients/python/lens/client/query.py b/contrib/clients/python/lens/client/query.py
index f82f0cb..df16cca 100644
--- a/contrib/clients/python/lens/client/query.py
+++ b/contrib/clients/python/lens/client/query.py
@@ -15,21 +15,26 @@
# limitations under the License.
#
import codecs
+import csv
+import logging
import time
import zipfile
import requests
-from six import string_types, BytesIO, StringIO, PY2, PY3
+from requests.exceptions import HTTPError
+from six import string_types, BytesIO, PY2, PY3
+
from .models import WrappedJson
from .utils import conf_to_xml
-import csv
+logger = logging.getLogger(__name__)
long_type = int
if PY3:
from collections.abc import Iterable as Iterable
elif PY2:
from collections import Iterable as Iterable
+
long_type = long
@@ -70,6 +75,7 @@ type_mappings = {'BOOLEAN': bool,
}
default_mapping = lambda x: x
+
class LensQueryResult(Iterable):
def __init__(self, custom_mappings=None):
if custom_mappings is None:
@@ -93,6 +99,7 @@ class LensInMemoryResult(LensQueryResult):
for row in self.rows:
yield list(self._mapping(value.type)(value.value) if value else None for value in row['values'])
+
class LensPersistentResult(LensQueryResult):
def __init__(self, header, response, encoding=None, is_header_present=True, delimiter=",",
custom_mappings=None):
@@ -107,6 +114,11 @@ class LensPersistentResult(LensQueryResult):
def _parse_line(self, line):
return list(self._mapping(self.header.columns[index].type)(line[index]) for index in range(len(line)))
+ def get_csv_reader(self, file):
+ if PY3:
+ file = codecs.iterdecode(file, 'utf-8')
+ return csv.reader(file, delimiter=self.delimiter)
+
def __iter__(self):
if self.is_zipped:
byte_stream = BytesIO(self.response.content)
@@ -114,7 +126,7 @@ class LensPersistentResult(LensQueryResult):
for name in self.zipfile.namelist():
with self.zipfile.open(name) as single_file:
if name[-3:] == 'csv':
- reader = csv.reader(single_file, delimiter=self.delimiter)
+ reader = self.get_csv_reader(single_file)
else:
reader = single_file
reader_iterator = iter(reader)
@@ -142,7 +154,8 @@ class LensQueryClient(object):
self.launched_queries = []
self.finished_queries = {}
self.query_confs = {}
- self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup']
+ self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() \
+ in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup']
def __call__(self, **filters):
filters['sessionid'] = self._session._sessionid
@@ -183,6 +196,7 @@ class LensQueryClient(object):
payload.append(('conf', conf_to_xml(conf)))
resp = requests.post(self.base_url + "queries/", files=payload, headers={'accept': 'application/json'})
query = self.sanitize_response(resp)
+ logger.info("Submitted query %s", query)
if conf:
self.query_confs[str(query)] = conf
if fetch_result:
@@ -216,7 +230,8 @@ class LensQueryClient(object):
return LensPersistentResult(metadata, resp, is_header_present=is_header_present, *args, **kwargs)
else:
response = requests.get(self.base_url + "queries/" + handle + "/resultset",
- params={'sessionid': self._session._sessionid}, headers={'accept': 'application/json'})
+ params={'sessionid': self._session._sessionid},
+ headers={'accept': 'application/json'})
resp = self.sanitize_response(response)
# If it has in memory result, return inmemory result iterator
if resp._is_wrapper and resp._wrapped_key == u'inMemoryQueryResult':
@@ -228,13 +243,18 @@ class LensQueryClient(object):
raise Exception("Result set not available")
def sanitize_response(self, resp):
- resp.raise_for_status()
try:
resp_json = resp.json(object_hook=WrappedJson)
+ except:
+ resp_json = resp.json()
+ if resp_json is not None:
if 'lensAPIResult' in resp_json:
resp_json = resp_json.lens_a_p_i_result
if 'error' in resp_json:
- raise Exception(resp_json['error'])
+ error = resp_json['error']
+ if "stackTrace" in error:
+ logger.error(error['stackTrace'])
+ raise HTTPError(error, request=resp.request, response=resp)
if 'data' in resp_json:
data = resp_json.data
if len(data) == 2 and 'type' in data:
@@ -242,6 +262,9 @@ class LensQueryClient(object):
keys.remove('type')
return WrappedJson({data['type']: data[keys[0]]})
return data
- except:
- resp_json = resp.json()
- return resp_json
+ if resp_json is not None:
+ return resp_json
+ else:
+ resp.raise_for_status()
+ logger.error(resp.text)
+ raise Exception("Unknown error with response", resp)
http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/setup.py
----------------------------------------------------------------------
diff --git a/contrib/clients/python/setup.py b/contrib/clients/python/setup.py
index 113205d..de59d32 100644
--- a/contrib/clients/python/setup.py
+++ b/contrib/clients/python/setup.py
@@ -45,6 +45,8 @@ class Tox(TestCommand):
def run_tests(self):
#import here, cause outside the eggs aren't loaded
import tox
+ import os
+ del os.environ["PYTHONPATH"]
errcode = tox.cmdline(self.test_args)
sys.exit(errcode)
http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/test/test_lensclient.py
----------------------------------------------------------------------
diff --git a/contrib/clients/python/test/test_lensclient.py b/contrib/clients/python/test/test_lensclient.py
index e9b0bdb..b8d7c73 100644
--- a/contrib/clients/python/test/test_lensclient.py
+++ b/contrib/clients/python/test/test_lensclient.py
@@ -29,7 +29,9 @@ from lens.client.models import WrappedJson
from requests.exceptions import HTTPError
from lens.client import LensClient
-
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(level="DEBUG")
def check_output(command):
output = subprocess.check_output(command.split())
@@ -85,11 +87,14 @@ class TestLensClient(object):
cls.base_path = glob.glob(joined)[0]
with cwd(cls.base_path):
with cwd('server'):
+ logger.info("Inside server directory")
server_start_output = check_output("bin/lens-ctl restart")
assert "Started lens server" in server_start_output
assert os.path.exists('logs/server.pid')
+ logger.info("started lens server")
time.sleep(1)
while not os.path.exists('logs/lensserver.log'):
+ logger.info("waiting for lensserver.log to be created")
error = get_error()
if has_error(error):
# Assert again with complete error
@@ -97,9 +102,11 @@ class TestLensClient(object):
time.sleep(1)
error = get_error()
if has_error(error):
+ logger.error(error)
assert False, error
-
+ logger.info("finished setting up server environment. Will setup client now")
with cwd('client'):
+ logger.info("Inside client directory")
cls.candidate_query = select_query('examples/resources/cube-queries.sql')
with open('check_connection.sql', 'w') as f:
f.write('show databases')
@@ -160,7 +167,7 @@ class TestLensClient(object):
handle = client.queries.submit(self.candidate_query, query_name="Candidate Query")
finished_query = client.queries.wait_till_finish(handle)
assert client.queries[handle] == finished_query
- queries = client.queries(state='SUCCESSFUL', fromDate=finished_query.submission_time - 1,
+ queries = client.queries(state='successful,failed', fromDate=finished_query.submission_time - 1,
toDate=finished_query.submission_time + 1)
assert handle in queries
http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/tools/conf/server/lens-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf/server/lens-site.xml b/tools/conf/server/lens-site.xml
index b53ff25..c303198 100644
--- a/tools/conf/server/lens-site.xml
+++ b/tools/conf/server/lens-site.xml
@@ -36,4 +36,10 @@
<name>datanucleus.schema.autoCreateTables</name>
<value>true</value>
</property>
+
+<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
</configuration>