You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/14 06:02:47 UTC
[impala] branch master updated: IMPALA-8515: port shell tests to
use shell build
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new b55d905 IMPALA-8515: port shell tests to use shell build
b55d905 is described below
commit b55d905322db017a11b5424da9c26c8d43aebb4c
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Mon May 6 11:08:46 2019 -0700
IMPALA-8515: port shell tests to use shell build
shell/make_shell_tarball.sh builds a tarball with all the
shell dependencies bundled. We should test the contents of
that tarball in the shell tests instead of using infra/python/env
and the libraries bundled there.
This tarball is one of the default targets (e.g. run by buildall.sh) so
this should not affect any typical development workflows.
Note that this means the shell tests now requires the shell tarball to
be built locally, which doesn't necessarily happen for remote cluster
tests, so we preserve the old behaviour in that case.
Testing:
Ran core tests on CentOS 6 and CentOS 7.
Change-Id: I581363639b279a9c2ff1fd982bdb140260b24baa
Reviewed-on: http://gerrit.cloudera.org:8080/13267
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/common/environ.py | 12 ++++++++++++
tests/shell/test_shell_commandline.py | 6 +++---
tests/shell/util.py | 26 ++++++++++++++++++++++++--
3 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 6e032f5..5b92755 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -29,6 +29,18 @@ IMPALA_REMOTE_URL = os.environ.get("IMPALA_REMOTE_URL", "")
# Default web UI URL for local test cluster
DEFAULT_LOCAL_WEB_UI_URL = "http://localhost:25000"
+# Find the local build version. May be None if Impala wasn't built locally.
+IMPALA_LOCAL_BUILD_VERSION = None
+IMPALA_LOCAL_VERSION_INFO = os.path.join(IMPALA_HOME, "bin/version.info")
+if os.path.isfile(IMPALA_LOCAL_VERSION_INFO):
+ with open(IMPALA_LOCAL_VERSION_INFO) as f:
+ for line in f:
+ match = re.match("VERSION: ([^\s]*)\n", line)
+ if match:
+ IMPALA_LOCAL_BUILD_VERSION = match.group(1)
+ if IMPALA_LOCAL_BUILD_VERSION is None:
+ raise Exception("Could not find VERSION in {0}".format(IMPALA_LOCAL_VERSION_INFO))
+
# Find the likely BuildType of the running Impala. Assume it's found through the path
# $IMPALA_HOME/be/build/latest as a fallback.
build_type_arg_regex = re.compile(r'--build_type=(\w+)', re.I)
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 11be9a8..cfef7c6 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -30,8 +30,8 @@ from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_POR
from tests.common.skip import SkipIf
from tests.common.test_dimensions import create_beeswax_dimension
from time import sleep, time
-from util import get_impalad_host_port
-from util import assert_var_substitution, run_impala_shell_cmd, ImpalaShell
+from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
+ ImpalaShell, IMPALA_SHELL_EXECUTABLE)
from contextlib import closing
DEFAULT_QUERY = 'select 1'
@@ -707,7 +707,7 @@ class TestImpalaShell(ImpalaTestSuite):
def _validate_expected_socket_connected(self, vector, args, sock):
# Building an one-off shell command instead of using Util::ImpalaShell since we need
# to customize the impala daemon socket.
- shell_cmd = ["{0}/bin/impala-shell.sh".format(os.environ['IMPALA_HOME'])]
+ shell_cmd = [IMPALA_SHELL_EXECUTABLE]
expected_output = "PingImpalaService"
with open(os.devnull, 'w') as devnull:
try:
diff --git a/tests/shell/util.py b/tests/shell/util.py
index 6c15250..80c8d9d 100755
--- a/tests/shell/util.py
+++ b/tests/shell/util.py
@@ -25,8 +25,23 @@ import shlex
import time
from subprocess import Popen, PIPE
+from tests.common.environ import (IMPALA_LOCAL_BUILD_VERSION,
+ IMPALA_TEST_CLUSTER_PROPERTIES)
from tests.common.impala_test_suite import IMPALAD_BEESWAX_HOST_PORT
+
+
SHELL_HISTORY_FILE = os.path.expanduser("~/.impalahistory")
+IMPALA_HOME = os.environ['IMPALA_HOME']
+
+if IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster():
+ # With remote cluster testing, we cannot assume that the shell was built locally.
+ IMPALA_SHELL_EXECUTABLE = os.path.join(IMPALA_HOME, "bin/impala-shell.sh")
+else:
+ # Test the locally built shell distribution.
+ IMPALA_SHELL_EXECUTABLE = os.path.join(
+ IMPALA_HOME, "shell/build", "impala-shell-" + IMPALA_LOCAL_BUILD_VERSION,
+ "impala-shell")
+
def assert_var_substitution(result):
assert_pattern(r'\bfoo_number=.*$', 'foo_number= 123123', result.stdout, \
@@ -129,8 +144,10 @@ def get_impalad_port(vector):
def get_shell_cmd(vector):
"""Get the basic shell command to start the shell, given the provided test vector.
Returns the command as a list of string arguments."""
- return [os.path.join(os.environ['IMPALA_HOME'], "bin/impala-shell.sh"),
- "-i{0}".format(get_impalad_host_port(vector))]
+ # Use impala-shell build instead of bin/impala-shell.sh so that we test with the
+ # system python, not the toolchain python and in a configuration close to what
+ # we will distribute.
+ return [IMPALA_SHELL_EXECUTABLE, "-i{0}".format(get_impalad_host_port(vector))]
def get_open_sessions_metric(vector):
@@ -206,5 +223,10 @@ class ImpalaShell(object):
cmd = get_shell_cmd(vector)
if args is not None: cmd += args
if not env: env = os.environ
+ # Don't inherit PYTHONPATH - the shell launch script should set up PYTHONPATH
+ # to include dependencies. Copy 'env' to avoid mutating argument or os.environ.
+ env = dict(env)
+ if "PYTHONPATH" in env:
+ del env["PYTHONPATH"]
return Popen(cmd, shell=False, stdout=PIPE, stdin=PIPE, stderr=PIPE,
env=env)