You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2018/12/17 15:12:44 UTC
[arrow] branch master updated: ARROW-4045: [Packaging/Python] Add
hypothesis test dependency to wheel crossbow tests
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 4cfd6d3 ARROW-4045: [Packaging/Python] Add hypothesis test dependency to wheel crossbow tests
4cfd6d3 is described below
commit 4cfd6d3877e28624e271e022f7c98a8b1e3c5a5a
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Mon Dec 17 16:12:36 2018 +0100
ARROW-4045: [Packaging/Python] Add hypothesis test dependency to wheel crossbow tests
Test builds: [kszucs/crossbow/build-383](https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=build-383)
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #3188 from kszucs/ARROW-4045 and squashes the following commits:
a2bcdaf4 <Krisztián Szűcs> correct path
11093d97 <Krisztián Szűcs> missing cython on osx
f06fb949 <Krisztián Szűcs> pin numpy version in appveyor.yml
cabedfba <Krisztián Szűcs> remove last pandas version from tasks.yml
5309a5b2 <Krisztián Szűcs> requirements-wheel
97bc6ead <Krisztián Szűcs> fix requirements.txt path on osx
c17d6748 <Krisztián Szűcs> win
aa05e743 <Krisztián Szűcs> linux
6be30182 <Krisztián Szűcs> osx
---
dev/release/rat_exclude_files.txt | 1 +
dev/release/verify-release-candidate.sh | 2 +-
dev/tasks/python-wheels/appveyor.yml | 2 +-
dev/tasks/python-wheels/linux-test.sh | 2 +-
dev/tasks/python-wheels/osx-build.sh | 12 ++++--------
dev/tasks/python-wheels/travis.linux.yml | 1 -
dev/tasks/python-wheels/travis.osx.yml | 2 --
dev/tasks/python-wheels/win-build.bat | 2 +-
dev/tasks/tasks.yml | 20 --------------------
python/manylinux1/build_arrow.sh | 11 +++--------
python/requirements-test.txt | 1 -
python/requirements-wheel.txt | 4 ++++
12 files changed, 16 insertions(+), 44 deletions(-)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index e274d97..f2e3f16 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -130,6 +130,7 @@ python/pyarrow/includes/__init__.pxd
python/pyarrow/tests/__init__.py
python/requirements.txt
python/requirements-test.txt
+python/requirements-wheel.txt
pax_global_header
MANIFEST.in
__init__.pxd
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 45404b0..71324ec 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -189,7 +189,7 @@ test_and_install_cpp() {
test_python() {
pushd python
- pip install -r requirements-test.txt
+ pip install -r requirements.txt -r requirements-test.txt
python setup.py build_ext --inplace --with-parquet --with-plasma
py.test pyarrow -v --pdb
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index 016041a..c220f92 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -20,7 +20,7 @@ os: Visual Studio 2015
environment:
ARCH: "64"
GENERATOR: Visual Studio 14 2015 Win64
- NUMPY: "{{ numpy_version }}"
+ NUMPY: "1.14.5"
PYTHON: "{{ python_version }}"
MSVC_DEFAULT_OPTIONS: ON
ARROW_SRC: C:\apache-arrow
diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/linux-test.sh
index 163730a..234ce8d 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/dev/tasks/python-wheels/linux-test.sh
@@ -30,5 +30,5 @@ python -c "import pyarrow.parquet"
python -c "import pyarrow.plasma"
# Run pyarrow tests
-pip install pytest pandas
+pip install -r /arrow/python/requirements-test.txt
pytest --pyargs pyarrow
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 5c69904..22c44c1 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -99,9 +99,8 @@ function build_wheel {
# build will also work with newer NumPy versions.
export ARROW_HOME=`pwd`/arrow-dist
export PARQUET_HOME=`pwd`/arrow-dist
- if [ -n "$BUILD_DEPENDS" ]; then
- pip install $(pip_opts) $BUILD_DEPENDS
- fi
+
+ pip install $(pip_opts) -r python/requirements-wheel.txt cython
pushd cpp
mkdir build
@@ -161,10 +160,6 @@ function install_run {
wheelhouse="$PWD/python/dist"
- # Install test dependencies and built wheel
- if [ -n "$TEST_DEPENDS" ]; then
- pip install $(pip_opts) $TEST_DEPENDS
- fi
# Install compatible wheel
pip install $(pip_opts) \
$(python $multibuild_dir/supported_wheels.py $wheelhouse/*.whl)
@@ -179,7 +174,8 @@ function install_run {
python -c "import pyarrow.plasma"
# Run pyarrow tests
- pip install pytest pytest-faulthandler
+ pip install $(pip_opts) -r python/requirements-test.txt
+
py.test --pyargs pyarrow
popd
diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml
index 9a8f804..17888cc 100644
--- a/dev/tasks/python-wheels/travis.linux.yml
+++ b/dev/tasks/python-wheels/travis.linux.yml
@@ -42,7 +42,6 @@ script:
- docker run --shm-size=2g
-e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.version }}
-e PYTHON_VERSIONS="{{ python_version }},{{ unicode_width }}"
- -e WHEEL_VERSION={{ wheel_version }}
-v $PWD:/io
-v $PWD/../../:/arrow
quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index 2f0d168..c6bd010 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -29,8 +29,6 @@ env:
- PYARROW_VERSION={{ arrow.version }}
- PYARROW_BUILD_VERBOSE=1
- MB_PYTHON_VERSION={{ python_version }}
- - BUILD_DEPENDS="wheel=={{ wheel_version }} numpy=={{ numpy_version }} cython==0.27.3 six"
- - TEST_DEPENDS="numpy=={{ numpy_version }} pandas=={{ pandas_version }} six"
before_install:
- git clone https://github.com/matthew-brett/multibuild # TODO pin it
diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat
index 22e306a..f85c8e8 100644
--- a/dev/tasks/python-wheels/win-build.bat
+++ b/dev/tasks/python-wheels/win-build.bat
@@ -82,7 +82,7 @@ popd
@rem test the wheel
call deactivate
conda create -n wheel-test -q -y python=%PYTHON% ^
- numpy=%NUMPY% pandas pytest
+ numpy=%NUMPY% pandas pytest hypothesis
call activate wheel-test
pip install --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index bd49616..ea104d5 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -146,7 +146,6 @@ tasks:
platform: linux
template: python-wheels/travis.linux.yml
params:
- wheel_version: 0.31.1
python_version: 2.7
unicode_width: 16
test_docker_images: []
@@ -157,7 +156,6 @@ tasks:
platform: linux
template: python-wheels/travis.linux.yml
params:
- wheel_version: 0.31.1
python_version: 2.7
unicode_width: 32
test_docker_images:
@@ -169,7 +167,6 @@ tasks:
platform: linux
template: python-wheels/travis.linux.yml
params:
- wheel_version: 0.31.1
python_version: 3.5
unicode_width: 16
test_docker_images:
@@ -181,7 +178,6 @@ tasks:
platform: linux
template: python-wheels/travis.linux.yml
params:
- wheel_version: 0.31.1
python_version: 3.6
unicode_width: 16
test_docker_images:
@@ -193,7 +189,6 @@ tasks:
platform: linux
template: python-wheels/travis.linux.yml
params:
- wheel_version: 0.31.1
python_version: 3.7
unicode_width: 16
test_docker_images:
@@ -207,10 +202,7 @@ tasks:
platform: osx
template: python-wheels/travis.osx.yml
params:
- numpy_version: 1.14.5
- pandas_version: 0.23.0
python_version: 2.7
- wheel_version: 0.31.1
artifacts:
- pyarrow-{version}-cp27-cp27m-macosx_10_6_intel.whl
@@ -218,10 +210,7 @@ tasks:
platform: osx
template: python-wheels/travis.osx.yml
params:
- numpy_version: 1.14.5
- pandas_version: 0.23.0
python_version: 3.5
- wheel_version: 0.31.1
artifacts:
- pyarrow-{version}-cp35-cp35m-macosx_10_6_intel.whl
@@ -229,10 +218,7 @@ tasks:
platform: osx
template: python-wheels/travis.osx.yml
params:
- numpy_version: 1.14.5
- pandas_version: 0.23.0
python_version: 3.6
- wheel_version: 0.31.1
artifacts:
- pyarrow-{version}-cp36-cp36m-macosx_10_6_intel.whl
@@ -240,10 +226,7 @@ tasks:
platform: osx
template: python-wheels/travis.osx.yml
params:
- numpy_version: 1.14.5
- pandas_version: 0.23.0
python_version: 3.7
- wheel_version: 0.31.1
artifacts:
- pyarrow-{version}-cp37-cp37m-macosx_10_6_intel.whl
@@ -253,7 +236,6 @@ tasks:
platform: win
template: python-wheels/appveyor.yml
params:
- numpy_version: 1.14.5
python_version: 3.5
artifacts:
- pyarrow-{version}-cp35-cp35m-win_amd64.whl
@@ -262,7 +244,6 @@ tasks:
platform: win
template: python-wheels/appveyor.yml
params:
- numpy_version: 1.14.5
python_version: 3.6
artifacts:
- pyarrow-{version}-cp36-cp36m-win_amd64.whl
@@ -271,7 +252,6 @@ tasks:
platform: win
template: python-wheels/appveyor.yml
params:
- numpy_version: 1.14.5
python_version: 3.7
artifacts:
- pyarrow-{version}-cp37-cp37m-win_amd64.whl
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 9042973..b1d8f85 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -64,11 +64,6 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
fi
fi
- # pin wheel, because auditwheel is not compatible with wheel=0.32
- # pin after installing tensorflow, because it updates to wheel=0.32
- # TODO(kszucs): remove after auditwheel properly supports wheel>0.31
- $PIP install "wheel==${WHEEL_VERSION:-0.31.1}"
-
echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
ARROW_BUILD_DIR=/tmp/build-PY${PYTHON}-${U_WIDTH}
mkdir -p "${ARROW_BUILD_DIR}"
@@ -96,6 +91,9 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
# Check that we don't expose any unwanted symbols
/io/scripts/check_arrow_visibility.sh
+ echo "=== (${PYTHON}) Install the wheel build dependencies ==="
+ $PIP install -r requirements-wheel.txt
+
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
@@ -107,9 +105,6 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist
- echo "=== (${PYTHON}) Ensure the existence of mandatory modules ==="
- $PIP install -r requirements.txt
-
echo "=== (${PYTHON}) Tag the wheel with manylinux1 ==="
mkdir -p repaired_wheels/
auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/
diff --git a/python/requirements-test.txt b/python/requirements-test.txt
index 482e888..89af5ec 100644
--- a/python/requirements-test.txt
+++ b/python/requirements-test.txt
@@ -1,4 +1,3 @@
--r requirements.txt
pandas
pytest
hypothesis
diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt
new file mode 100644
index 0000000..c44903e
--- /dev/null
+++ b/python/requirements-wheel.txt
@@ -0,0 +1,4 @@
+wheel==0.31.1
+six>=1.0.0
+numpy==1.14.5
+futures; python_version < "3.2"