You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2017/08/12 16:14:37 UTC
kudu git commit: python: restore setuptools requirement and employ
different workaround
Repository: kudu
Updated Branches:
refs/heads/master ce6ecc5a5 -> b198ed8ff
python: restore setuptools requirement and employ different workaround
Commit d0acb55 moved the installation of setuptools from requirements.txt
to build-and-test.sh itself. This is somewhat inconvenient, and it reduces
the usefulness of requirements.txt; it's no longer a "one stop shop" for all
dependencies needed to build the Python bindings.
The build problems we saw were due to a virtualenv like so:
1. Initially, an old pip and setuptools
2. An upgraded pip (via `pip install --upgrade pip` in build-and-test.sh)
When `pip install -r requirements.txt` is run with such a virtualenv, the
new pip tries to upgrade the old setuptools and is unable to do so.
Let's try a different approach: let's not upgrade pip at all. We'll start
with whatever pip/setuptools are in the virtualenv, and we'll use that pip
to upgrade setuptools via the usual `pip install -r requirements.txt`. This
appears to work on both CentOS 6.6 and Ubuntu 16.04. On CentOS 6.6 I tested
with both python-virtualenv 1.7.2 (which initializes a virtualenv with
pip 1.1 and setuptools 0.6c11) and 1.10.1 (pip 1.4.1 and setuptools 0.9.8).
Change-Id: I1a8947bc8e89ec73728749bbcc6e9b919e26838a
Reviewed-on: http://gerrit.cloudera.org:8080/7661
Tested-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Jean-Daniel Cryans <jd...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b198ed8f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b198ed8f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b198ed8f
Branch: refs/heads/master
Commit: b198ed8ff6a603816892c8bc7fd80679a014aaf1
Parents: ce6ecc5
Author: Adar Dembo <ad...@cloudera.com>
Authored: Fri Aug 11 16:32:08 2017 -0700
Committer: Jean-Daniel Cryans <jd...@apache.org>
Committed: Sat Aug 12 15:50:15 2017 +0000
----------------------------------------------------------------------
build-support/jenkins/build-and-test.sh | 42 +++++++++++++++-------------
python/requirements.txt | 1 +
2 files changed, 24 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/b198ed8f/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index d2e9a94..7789bd3 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -273,6 +273,7 @@ set +e
# Run tests
export GTEST_OUTPUT="xml:$TEST_LOGDIR/" # Enable JUnit-compatible XML output.
+mkdir -p $TEST_LOGDIR # In case no tests are run (the directory won't be created)
if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
if [ -z "$TEST_RESULT_SERVER" ]; then
echo Must set TEST_RESULT_SERVER to use RUN_FLAKY_ONLY
@@ -380,34 +381,35 @@ if [ "$BUILD_PYTHON" == "1" ]; then
echo Building and testing python.
echo ------------------------------------------------------------
- # Failing to compile the Python client should result in a build failure
+ # Failing to compile the Python client should result in a build failure.
set -e
export KUDU_HOME=$SOURCE_ROOT
export KUDU_BUILD=$BUILD_ROOT
pushd $SOURCE_ROOT/python
- # Create a sane test environment
+ # Create a sane test environment.
rm -Rf $KUDU_BUILD/py_env
virtualenv $KUDU_BUILD/py_env
source $KUDU_BUILD/py_env/bin/activate
- pip install --upgrade pip
- # On RHEL 6, keeping setuptools in requirements.txt fails as of setuptools 36.2.7.
- # Strangely, it does work when extracted like this.
- pip install --disable-pip-version-check --upgrade 'setuptools >= 0.8'
- CC=$CLANG CXX=$CLANG++ pip install --disable-pip-version-check -r requirements.txt
+
+ # Download all necessary requirements.
+ CC=$CLANG CXX=$CLANG++ pip install -r requirements.txt
# Delete old Cython extensions to force them to be rebuilt.
rm -Rf build kudu_python.egg-info kudu/*.so
- # Assuming we run this script from base dir
+ # Build the Python bindings. This assumes we run this script from base dir.
CC=$CLANG CXX=$CLANG++ python setup.py build_ext
set +e
+
+ # Run the Python tests.
if ! python setup.py test \
- --addopts="kudu --junit-xml=$KUDU_BUILD/test-logs/python_client.xml" \
- 2> $KUDU_BUILD/test-logs/python_client.log ; then
+ --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
+ 2> $TEST_LOGDIR/python_client.log ; then
EXIT_STATUS=1
FAILURES="$FAILURES"$'Python tests failed\n'
fi
+
deactivate
popd
fi
@@ -417,33 +419,35 @@ if [ "$BUILD_PYTHON3" == "1" ]; then
echo Building and testing python 3.
echo ------------------------------------------------------------
- # Failing to compile the Python client should result in a build failure
+ # Failing to compile the Python client should result in a build failure.
set -e
export KUDU_HOME=$SOURCE_ROOT
export KUDU_BUILD=$BUILD_ROOT
pushd $SOURCE_ROOT/python
- # Create a sane test environment
+ # Create a sane test environment.
rm -Rf $KUDU_BUILD/py_env
virtualenv -p python3 $KUDU_BUILD/py_env
source $KUDU_BUILD/py_env/bin/activate
- pip install --upgrade pip
- # See the comment in the BUILD_PYTHON section above for why we need this line.
- pip install --disable-pip-version-check --upgrade 'setuptools >= 0.8'
- CC=$CLANG CXX=$CLANG++ pip install --disable-pip-version-check -r requirements.txt
+
+ # Download all necessary requirements.
+ CC=$CLANG CXX=$CLANG++ pip install -r requirements.txt
# Delete old Cython extensions to force them to be rebuilt.
rm -Rf build kudu_python.egg-info kudu/*.so
- # Assuming we run this script from base dir
+ # Build the Python bindings. This assumes we run this script from base dir.
CC=$CLANG CXX=$CLANG++ python setup.py build_ext
set +e
+
+ # Run the Python tests.
if ! python setup.py test \
- --addopts="kudu --junit-xml=$KUDU_BUILD/test-logs/python3_client.xml" \
- 2> $KUDU_BUILD/test-logs/python3_client.log ; then
+ --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
+ 2> $TEST_LOGDIR/python3_client.log ; then
EXIT_STATUS=1
FAILURES="$FAILURES"$'Python 3 tests failed\n'
fi
+
deactivate
popd
fi
http://git-wip-us.apache.org/repos/asf/kudu/blob/b198ed8f/python/requirements.txt
----------------------------------------------------------------------
diff --git a/python/requirements.txt b/python/requirements.txt
index 9df81cd..72d6c68 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,5 +1,6 @@
pytest
cython >= 0.21
+setuptools >= 0.8
six
unittest2
pytz