You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/09/14 18:56:29 UTC
[arrow] branch master updated: ARROW-3229: [Packaging]: Adjust
wheel package scripts to account for Parquet codebase migration
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c48eaec ARROW-3229: [Packaging]: Adjust wheel package scripts to account for Parquet codebase migration
c48eaec is described below
commit c48eaec92a4af62c5e6f02059b6ce787cffb9fc0
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Fri Sep 14 14:56:22 2018 -0400
ARROW-3229: [Packaging]: Adjust wheel package scripts to account for Parquet codebase migration
See the build results here: https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=build-308
Manylinux1 wheels are already [updated](https://github.com/apache/arrow/commit/9b4cd9c03ed9365f8e235f296caa166ea692c98f#diff-785c937d41d0bf0c426191906ee42039)
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #2552 from kszucs/ARROW-3229 and squashes the following commits:
6323fe7fa <Krisztián Szűcs> disable both arrow and 3rdparty verbose builds
7d5a0ede9 <Krisztián Szűcs> don't build thrift manually
f7e533131 <Krisztián Szűcs> adjust wheel builds
---
dev/tasks/python-wheels/appveyor.yml | 1 -
dev/tasks/python-wheels/osx-build.sh | 50 ++---------------------------------
dev/tasks/python-wheels/win-build.bat | 19 +------------
python/manylinux1/build_arrow.sh | 7 ++++-
4 files changed, 9 insertions(+), 68 deletions(-)
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index d9d208c..016041a 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -26,7 +26,6 @@ environment:
ARROW_SRC: C:\apache-arrow
PYARROW_VERSION: {{ arrow.version }}
PYARROW_REF: {{ arrow.head }}
- PARQUET_CPP_REF: master
init:
- set MINICONDA=C:\Miniconda35-x64
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
index 6dce16f..4170a93 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -93,37 +93,6 @@ function build_wheel {
install_name_tool -change libarrow_boost_system.dylib @rpath/libarrow_boost_system.dylib libarrow_boost_filesystem.dylib
popd
- # We build a custom version of thrift instead of using the one that comes
- # with brew as we also want it to use our namespaced version of Boost.
- # TODO(PARQUET-1262): Use the external project facilities of parquet-cpp.
- export THRIFT_HOME=`pwd`/thift-dist
- export THRIFT_VERSION=0.11.0
- wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
- tar xf thrift-${THRIFT_VERSION}.tar.gz
- pushd thrift-${THRIFT_VERSION}
- mkdir build-tmp
- pushd build-tmp
- cmake -DCMAKE_BUILD_TYPE=release \
- "-DCMAKE_CXX_FLAGS=-fPIC" \
- "-DCMAKE_C_FLAGS=-fPIC" \
- "-DCMAKE_INSTALL_PREFIX=${THRIFT_HOME}" \
- "-DCMAKE_INSTALL_RPATH=${THRIFT_HOME}/lib" \
- "-DBUILD_SHARED_LIBS=OFF" \
- "-DBUILD_TESTING=OFF" \
- "-DWITH_QT4=OFF" \
- "-DWITH_C_GLIB=OFF" \
- "-DWITH_JAVA=OFF" \
- "-DWITH_PYTHON=OFF" \
- "-DWITH_CPP=ON" \
- "-DWITH_STATIC_LIB=ON" \
- "-DWITH_LIBEVENT=OFF" \
- -DBoost_NAMESPACE=arrow_boost \
- -DBOOST_ROOT="$arrow_boost_dist" \
- ..
- make install -j5
- popd
- popd
-
# Now we can start with the actual build of Arrow and Parquet.
# We pin NumPy to an old version here as the NumPy version one builds
# with is the oldest supported one. Thanks to NumPy's guarantees our Arrow
@@ -131,6 +100,7 @@ function build_wheel {
export ARROW_HOME=`pwd`/arrow-dist
export PARQUET_HOME=`pwd`/arrow-dist
pip install "cython==0.27.3" "numpy==${NP_TEST_DEP}"
+
pushd cpp
mkdir build
pushd build
@@ -143,6 +113,7 @@ function build_wheel {
-DARROW_PLASMA=ON \
-DARROW_RPATH_ORIGIN=ON \
-DARROW_PYTHON=ON \
+ -DARROW_PARQUET=ON \
-DARROW_ORC=ON \
-DBOOST_ROOT="$arrow_boost_dist" \
-DBoost_NAMESPACE=arrow_boost \
@@ -153,23 +124,6 @@ function build_wheel {
popd
popd
- git clone https://github.com/apache/parquet-cpp.git
- pushd parquet-cpp
- mkdir build
- pushd build
- cmake -DCMAKE_BUILD_TYPE=Release \
- -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
- -DPARQUET_VERBOSE_THIRDPARTY_BUILD=ON \
- -DPARQUET_BUILD_TESTS=OFF \
- -DPARQUET_BOOST_USE_SHARED=ON \
- -DBoost_NAMESPACE=arrow_boost \
- -DBOOST_ROOT="$arrow_boost_dist" \
- ..
- make -j5 VERBOSE=1
- make install
- popd
- popd
-
# Unset the HOME variables and use pkg-config to discover the previously
# built binaries. By using pkg-config, we also are able to discover the
# ABI and SO versions of the dynamic libraries.
diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat
index 37d0b64..c73b39b 100644
--- a/dev/tasks/python-wheels/win-build.bat
+++ b/dev/tasks/python-wheels/win-build.bat
@@ -42,7 +42,6 @@ popd
set ARROW_HOME=%CONDA_PREFIX%\Library
set PARQUET_HOME=%CONDA_PREFIX%\Library
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
-set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
echo %ARROW_HOME%
@@ -57,6 +56,7 @@ cmake -G "%GENERATOR%" ^
-DCMAKE_BUILD_TYPE=Release ^
-DARROW_CXXFLAGS="/MP" ^
-DARROW_PYTHON=ON ^
+ -DARROW_PARQUET=ON ^
.. || exit /B
cmake --build . --target INSTALL --config Release || exit /B
@@ -65,23 +65,6 @@ set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX
ctest -VV || exit /B
popd
-@rem Build parquet-cpp
-git clone https://github.com/apache/parquet-cpp.git || exit /B
-pushd parquet-cpp
-git checkout "%PARQUET_CPP_REF%"
-popd
-
-mkdir parquet-cpp\build
-pushd parquet-cpp\build
-
-cmake -G "%GENERATOR%" ^
- -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
- -DCMAKE_BUILD_TYPE=Release ^
- -DPARQUET_BOOST_USE_SHARED=OFF ^
- -DPARQUET_BUILD_TESTS=OFF .. || exit /B
-cmake --build . --target INSTALL --config Release || exit /B
-popd
-
@rem Build and import pyarrow
set PYTHONPATH=
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 1313842..04034e7 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -90,7 +90,12 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
- PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp --bundle-boost --boost-namespace=arrow_boost
+ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext \
+ --inplace \
+ --with-parquet \
+ --bundle-arrow-cpp \
+ --bundle-boost \
+ --boost-namespace=arrow_boost
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist