You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:28:26 UTC
[17/42] incubator-spot git commit: [SPOT-213][SPOT-250][OA][DATA]
temp fix for impala calls, add TODO for impyla conversion
[SPOT-213][SPOT-250][OA][DATA] temp fix for impala calls, add TODO for impyla conversion
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0e749191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0e749191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0e749191
Branch: refs/heads/SPOT-181_ODM
Commit: 0e749191311a3c1695cd40322c1b5788cc56e50c
Parents: d1f5a67
Author: natedogs911 <na...@gmail.com>
Authored: Thu Jan 18 11:06:50 2018 -0800
Committer: natedogs911 <na...@gmail.com>
Committed: Thu Jan 18 11:06:50 2018 -0800
----------------------------------------------------------------------
spot-oa/oa/components/data/hive.py | 1 +
spot-oa/oa/components/data/impala.py | 25 +++++++++++++++++++++----
2 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0e749191/spot-oa/oa/components/data/hive.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/data/hive.py b/spot-oa/oa/components/data/hive.py
index a7c1d4b..7d2eaa2 100644
--- a/spot-oa/oa/components/data/hive.py
+++ b/spot-oa/oa/components/data/hive.py
@@ -24,6 +24,7 @@ class Engine(object):
self._pipeline = pipeline
def query(self,query,output_file=None, delimiter=','):
+ # TODO: fix kerberos compatibility, use impyla
hive_config = "set mapred.max.split.size=1073741824;set hive.exec.reducers.max=10;set hive.cli.print.header=true;"
del_format = "| sed 's/[\t]/{0}/g'".format(delimiter)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0e749191/spot-oa/oa/components/data/impala.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/data/impala.py b/spot-oa/oa/components/data/impala.py
index bfc1c5a..10d1f5b 100644
--- a/spot-oa/oa/components/data/impala.py
+++ b/spot-oa/oa/components/data/impala.py
@@ -16,6 +16,8 @@
#
from subprocess import check_output
+from common import configurator
+
class Engine(object):
@@ -24,17 +26,32 @@ class Engine(object):
self._daemon_node = conf['impala_daemon']
self._db = db
self._pipeline = pipeline
- impala_cmd = "impala-shell -i {0} --quiet -q 'INVALIDATE METADATA {1}.{2}'".format(self._daemon_node,self._db, self._pipeline)
+
+ if configurator.kerberos_enabled():
+ self._impala_shell = "impala-shell -k -i {0} --quiet".format(self._daemon_node)
+ else:
+ self._impala_shell = "impala-shell -i {0} --quiet".format(self._daemon_node)
+
+ impala_cmd = "{0} -q 'INVALIDATE METADATA {1}.{2}'".format(self._impala_shell, self._db, self._pipeline)
check_output(impala_cmd,shell=True)
- impala_cmd = "impala-shell -i {0} --quiet -q 'REFRESH {1}.{2}'".format(self._daemon_node,self._db, self._pipeline)
+ impala_cmd = "{0} -q 'REFRESH {1}.{2}'".format(self._impala_shell, self._db, self._pipeline)
check_output(impala_cmd,shell=True)
def query(self,query,output_file=None,delimiter=","):
if output_file:
- impala_cmd = "impala-shell -i {0} --quiet --print_header -B --output_delimiter='{1}' -q \"{2}\" -o {3}".format(self._daemon_node,delimiter,query,output_file)
+ impala_cmd = "{0} --print_header -B --output_delimiter='{1}' -q \"{2}\" -o {3}".format(
+ self._impala_shell,
+ delimiter,
+ query,
+ output_file
+ )
else:
- impala_cmd = "impala-shell -i {0} --quiet --print_header -B --output_delimiter='{1}' -q \"{2}\"".format(self._daemon_node,delimiter,query)
+ impala_cmd = "{0} --print_header -B --output_delimiter='{1}' -q \"{2}\"".format(
+ self._impala_shell,
+ delimiter,
+ query
+ )
check_output(impala_cmd,shell=True)