You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/21 17:02:32 UTC
[arrow] branch master updated: ARROW-5656: [Python][Packaging] Fix
macOS wheel builds, add Flight support
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 3a37bf2 ARROW-5656: [Python][Packaging] Fix macOS wheel builds, add Flight support
3a37bf2 is described below
commit 3a37bf29c512b4c72c8da5b2a8657b21548cc47a
Author: Wes McKinney <we...@apache.org>
AuthorDate: Fri Jun 21 12:02:24 2019 -0500
ARROW-5656: [Python][Packaging] Fix macOS wheel builds, add Flight support
This fixes a regression introduced by the shared library duplication issue that was fixed in Linux recently.
Author: Wes McKinney <we...@apache.org>
Author: Philipp Moritz <pc...@gmail.com>
Closes #4642 from wesm/ARROW-5656 and squashes the following commits:
71e277664 <Wes McKinney> Review feedback, try to fix Python 3.5 windows wheels
7123ed6cb <Wes McKinney> Fix git submodule call
7e9d2c565 <Wes McKinney> Don't add ABI version to Boost dylibs on macOS for now
53dc41530 <Wes McKinney> cmake-format
1fe05a9be <Wes McKinney> Get gRPC using SYSTEM method, set ARROW_TEST_DATA env variable
497ba3f33 <Wes McKinney> Use gRPC from Homebrew
4f583e8f5 <Philipp Moritz> install requests package
a4db1ac6b <Wes McKinney> Use requests in get_apache_mirror.py if available
db0caab28 <Wes McKinney> First attempt to enable Flight in macOS wheels
---
cpp/Brewfile | 1 +
cpp/build-support/get_apache_mirror.py | 19 ++++++++++++++-----
dev/tasks/python-wheels/osx-build.sh | 11 ++++++++++-
dev/tasks/python-wheels/travis.osx.yml | 9 ++++++++-
dev/tasks/python-wheels/win-build.bat | 5 ++++-
python/CMakeLists.txt | 2 +-
python/requirements-wheel.txt | 1 +
7 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/cpp/Brewfile b/cpp/Brewfile
index c658887..4b796db 100644
--- a/cpp/Brewfile
+++ b/cpp/Brewfile
@@ -35,6 +35,7 @@ brew "python"
brew "rapidjson"
brew "re2"
brew "snappy"
+brew "openssl"
brew "thrift"
brew "wget"
brew "zstd"
diff --git a/cpp/build-support/get_apache_mirror.py b/cpp/build-support/get_apache_mirror.py
index ac52255..38ea6f4 100755
--- a/cpp/build-support/get_apache_mirror.py
+++ b/cpp/build-support/get_apache_mirror.py
@@ -21,11 +21,20 @@
import json
try:
- from urllib2 import urlopen
+ import requests
+
+ def get_url(url):
+ return requests.get(url).content
except ImportError:
- # py3
- from urllib.request import urlopen
+ try:
+ from urllib2 import urlopen
+ except ImportError:
+ # py3
+ from urllib.request import urlopen
+
+ def get_url(url):
+ return urlopen(url).read()
-suggested_mirror = urlopen('https://www.apache.org/dyn/'
- 'closer.cgi?as_json=1').read()
+suggested_mirror = get_url('https://www.apache.org/dyn/'
+ 'closer.cgi?as_json=1')
print(json.loads(suggested_mirror.decode('utf-8'))['preferred'])
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 9fef5b9..be39754 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -22,7 +22,9 @@ set -e
# overrides multibuild's default build_wheel
function build_wheel {
pip install -U pip
- pip install setuptools_scm
+
+ # ARROW-5670: Python 3.5 can fail with HTTPS error in CMake build
+ pip install setuptools_scm requests
# Include brew installed versions of flex and bison.
# We need them to build Thrift. The ones that come with Xcode are too old.
@@ -111,6 +113,9 @@ function build_wheel {
export BUILD_ARROW_GANDIVA=OFF
fi
+ git submodule update --init
+ export ARROW_TEST_DATA=`pwd`/testing/data
+
pushd cpp
mkdir build
pushd build
@@ -129,6 +134,8 @@ function build_wheel {
-DARROW_ORC=ON \
-DBOOST_ROOT="$arrow_boost_dist" \
-DBoost_NAMESPACE=arrow_boost \
+ -DARROW_FLIGHT=ON \
+ -DgRPC_SOURCE=SYSTEM \
-DMAKE=make \
..
make -j5
@@ -143,6 +150,7 @@ function build_wheel {
unset ARROW_HOME
unset PARQUET_HOME
+ export PYARROW_WITH_FLIGHT=1
export PYARROW_WITH_PLASMA=1
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_ORC=1
@@ -186,6 +194,7 @@ import pyarrow.parquet
import pyarrow.plasma
if sys.version_info.major > 2:
+ import pyarrow.flight
import pyarrow.gandiva
"
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index 919bcf5..2db3208 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -38,17 +38,24 @@ before_install:
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- git -C arrow checkout {{ arrow.head }}
+ # ARROW-3976 Old versions of git can cause failures when Homebrew prints a
+ # donation solicitation. Attempt to update git
+ - git --version
+ - brew upgrade git
+
# Also remove artifacts that depend on Boost
- brew uninstall boost cgal postgis sfcgal
- brew update
- brew upgrade cmake
- - brew install bison flex llvm@7
+ - brew install bison flex grpc openssl llvm@7
- export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh
- source multibuild/common_utils.sh
- source multibuild/travis_osx_steps.sh
- before_install
+ # Fix SSL TLS issue for Python 3.5 on macOS
+ - pip install requests[security]
install:
- mkdir -p dist
diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat
index a4ca71b..56b0097 100644
--- a/dev/tasks/python-wheels/win-build.bat
+++ b/dev/tasks/python-wheels/win-build.bat
@@ -23,13 +23,16 @@ conda update --yes --quiet conda
call conda create -n wheel-build -q -y -c conda-forge ^
--file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
--file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
- --file=%ARROW_SRC%\ci\conda_env_python.yml ^
python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% ^
|| exit /B
call activate wheel-build
+@rem Cannot use conda_env_python.yml here because conda-forge has
+@rem ceased providing up-to-date packages for Python 3.5
+pip install -r %ARROW_SRC%\python\requirements-wheel.txt
+
set ARROW_HOME=%CONDA_PREFIX%\Library
set PARQUET_HOME=%CONDA_PREFIX%\Library
echo %ARROW_HOME%
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 2a4d75c..6b9996b 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -261,7 +261,7 @@ function(bundle_boost_lib library_path)
if(APPLE)
configure_file(
${${library_path}}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}.${Boost_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
else()
configure_file(
diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt
index 8cc4bab..4b27ade 100644
--- a/python/requirements-wheel.txt
+++ b/python/requirements-wheel.txt
@@ -1,3 +1,4 @@
+cython
wheel==0.31.1
setuptools_scm==3.2.0
six>=1.0.0