You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2023/03/01 14:56:43 UTC
[impala] 03/04: IMPALA-11955: Detect system Pythons
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9cc19eb5f67248dff8edf8ea0193ec4f029f8b2a
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Feb 28 10:23:53 2023 -0800
IMPALA-11955: Detect system Pythons
Detect system Pythons (2 and 3) during build configuration. Build the
impala-shell tarball only using available Python versions, and test
available versions.
Drops support for DISABLE_PYTHON3_TEST as it's now automatically
detected. If python3 is present on the system, it's expected to be
usable.
Testing:
- built in SLES 15 SP4 container with Python 3
Change-Id: Iba36d0feba163e1251c66a6a49121d4dac625afc
Reviewed-on: http://gerrit.cloudera.org:8080/19560
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
CMakeLists.txt | 12 +++++++-----
bin/impala-config.sh | 6 ++++++
shell/make_shell_tarball.sh | 32 +++++++++++++++++++-------------
tests/shell/util.py | 19 ++++++++++---------
4 files changed, 42 insertions(+), 27 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 611c5a533..cb35b393a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -514,12 +514,14 @@ add_custom_target(impala_python ALL
COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
)
-if(NOT $ENV{DISABLE_PYTHON3_TEST} EQUAL "")
- message(STATUS "DISABLE_PYTHON3_TEST is set, disabling Python3 virtualenv and tests")
- add_custom_target(impala_shell_pypi ALL DEPENDS shell_python2_install)
-else()
- add_custom_target(impala_shell_pypi ALL DEPENDS shell_python2_install shell_python3_install)
+set(IMPALA_PYTHON_INSTALLS "")
+if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} EQUAL "")
+ list(APPEND IMPALA_PYTHON_INSTALLS shell_python2_install)
+endif()
+if (NOT $ENV{IMPALA_SYSTEM_PYTHON3} EQUAL "")
+ list(APPEND IMPALA_PYTHON_INSTALLS shell_python3_install)
endif()
+add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS})
add_custom_target(notests_independent_targets DEPENDS
java cscope tarballs impala_python impala_shell_pypi
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 8e08a2fce..ae9a00a42 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -196,6 +196,10 @@ unset IMPALA_THRIFT_JAVA_URL
export IMPALA_THRIFT_PY_VERSION=0.16.0-p3
unset IMPALA_THRIFT_PY_URL
+# Find system python versions for testing
+export IMPALA_SYSTEM_PYTHON2=$(command -v python2)
+export IMPALA_SYSTEM_PYTHON3=$(command -v python3)
+
if [[ $OSTYPE == "darwin"* ]]; then
IMPALA_CYRUS_SASL_VERSION=2.1.26
unset IMPALA_CYRUS_SASL_URL
@@ -982,6 +986,8 @@ echo "IMPALA_ICEBERG_VERSION = $IMPALA_ICEBERG_VERSION"
echo "IMPALA_COS_VERSION = $IMPALA_COS_VERSION"
echo "IMPALA_OSS_VERSION = $IMPALA_OSS_VERSION"
echo "IMPALA_OBS_VERSION = $IMPALA_OBS_VERSION"
+echo "IMPALA_SYSTEM_PYTHON2 = $IMPALA_SYSTEM_PYTHON2"
+echo "IMPALA_SYSTEM_PYTHON3 = $IMPALA_SYSTEM_PYTHON3"
# Kerberos things. If the cluster exists and is kerberized, source
# the required environment. This is required for any hadoop tool to
diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh
index 6a2273b44..775a63c48 100755
--- a/shell/make_shell_tarball.sh
+++ b/shell/make_shell_tarball.sh
@@ -118,25 +118,31 @@ for MODULE in ${SHELL_HOME}/ext-py/*; do
continue;
fi
pushd ${MODULE} > /dev/null 2>&1
- echo "Cleaning up old build artifacts."
- rm -rf dist 2>&1 > /dev/null
- rm -rf build 2>&1 > /dev/null
- echo "Creating an egg for ${MODULE}"
- if [[ "$MODULE" == *"/bitarray"* ]]; then
- # Need to use setuptools to build egg for bitarray module
- python -c "import setuptools; exec(open('setup.py').read())" -q bdist_egg
- else
- python setup.py -q bdist_egg clean
+ if [ ! -z "${IMPALA_SYSTEM_PYTHON2:-}" ]; then
+ echo "Cleaning up old build artifacts."
+ rm -rf dist 2>&1 > /dev/null
+ rm -rf build 2>&1 > /dev/null
+ echo "Creating a Python 2 egg for ${MODULE}"
+ if [[ "$MODULE" == *"/bitarray"* ]]; then
+ # Need to use setuptools to build egg for bitarray module
+ ${IMPALA_SYSTEM_PYTHON2} -c "import setuptools; exec(open('setup.py').read())" \
+ -q bdist_egg
+ else
+ ${IMPALA_SYSTEM_PYTHON2} setup.py -q bdist_egg clean
+ fi
+ cp dist/*.egg ${TARBALL_ROOT}/ext-py2
fi
- cp dist/*.egg ${TARBALL_ROOT}/ext-py2
- if [ -z "${DISABLE_PYTHON3_TEST:-}" ]; then
+ if [ ! -z "${IMPALA_SYSTEM_PYTHON3:-}" ]; then
+ echo "Cleaning up old build artifacts."
rm -rf dist 2>&1 > /dev/null
rm -rf build 2>&1 > /dev/null
+ echo "Creating a Python 3 egg for ${MODULE}"
if [[ "$MODULE" == *"/bitarray"* ]]; then
# Need to use setuptools to build egg for bitarray module
- python3 -c "import setuptools; exec(open('setup.py').read())" -q bdist_egg
+ ${IMPALA_SYSTEM_PYTHON3} -c "import setuptools; exec(open('setup.py').read())" \
+ -q bdist_egg
else
- python3 setup.py -q bdist_egg clean
+ ${IMPALA_SYSTEM_PYTHON3} setup.py -q bdist_egg clean
fi
cp dist/*.egg ${TARBALL_ROOT}/ext-py3
fi
diff --git a/tests/shell/util.py b/tests/shell/util.py
index f369c646d..2097eabd3 100755
--- a/tests/shell/util.py
+++ b/tests/shell/util.py
@@ -337,16 +337,17 @@ def get_dev_impala_shell_executable():
def create_impala_shell_executable_dimension(dev_only=False):
_, include_pypi = get_dev_impala_shell_executable()
+ dimensions = []
+ if os.getenv("IMPALA_SYSTEM_PYTHON2"):
+ dimensions.append('dev')
+ if os.getenv("IMPALA_SYSTEM_PYTHON3"):
+ dimensions.append('dev3')
if include_pypi and not dev_only:
- if 'DISABLE_PYTHON3_TEST' in os.environ:
- return ImpalaTestDimension('impala_shell', 'dev', 'python2')
- else:
- return ImpalaTestDimension('impala_shell', 'dev', 'dev3', 'python2', 'python3')
- else:
- if 'DISABLE_PYTHON3_TEST' in os.environ:
- return ImpalaTestDimension('impala_shell', 'dev')
- else:
- return ImpalaTestDimension('impala_shell', 'dev', 'dev3')
+ if os.getenv("IMPALA_SYSTEM_PYTHON2"):
+ dimensions.append('python2')
+ if os.getenv("IMPALA_SYSTEM_PYTHON3"):
+ dimensions.append('python3')
+ return ImpalaTestDimension('impala_shell', *dimensions)
def get_impala_shell_executable(vector):