You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/01/23 10:18:45 UTC
[arrow] branch master updated: ARROW-3926: [Python] Add Gandiva
bindings to Python manylinux1 wheels
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 995c58b ARROW-3926: [Python] Add Gandiva bindings to Python manylinux1 wheels
995c58b is described below
commit 995c58b5f86ec3583739a3df4656945aac8ae610
Author: Uwe L. Korn <uw...@xhochy.com>
AuthorDate: Wed Jan 23 11:18:22 2019 +0100
ARROW-3926: [Python] Add Gandiva bindings to Python manylinux1 wheels
Author: Uwe L. Korn <uw...@xhochy.com>
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #3438 from xhochy/ARROW-3926 and squashes the following commits:
688ade31 <Krisztián Szűcs> don't test gandiva for python 2 in the packaging task
a2f71670 <Uwe L. Korn> Build Gandiva only for Python 3.6+
65d43ae0 <Krisztián Szűcs> test gandiva optional import in crossbow packaging task
8271b185 <Uwe L. Korn> ARROW-3926: Add Gandiva bindings to Python wheels
---
cpp/CMakeLists.txt | 3 +++
cpp/src/gandiva/precompiled/CMakeLists.txt | 4 +++-
dev/tasks/python-wheels/linux-test.sh | 15 +++++++++++----
python/CMakeLists.txt | 3 +++
python/manylinux1/build_arrow.sh | 17 +++++++++++++++++
python/setup.py | 2 ++
6 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e553d08..fa69670 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -349,6 +349,9 @@ Always OFF if building binaries"
"Include -static-libstdc++ -static-libgcc when linking with Gandiva static libraries"
OFF)
+ set(ARROW_GANDIVA_PC_CXX_FLAGS "" CACHE STRING
+ "Compiler flags to append when pre-compiling Gandiva operations")
+
#----------------------------------------------------------------------
# Advanced developer options
diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt
index eab0b90..ba87e05 100644
--- a/cpp/src/gandiva/precompiled/CMakeLists.txt
+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt
@@ -43,8 +43,10 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS})
-DNDEBUG # DCHECK macros not implemented in precompiled code
-fno-use-cxa-atexit # Workaround for unresolved __dso_handle
-O3 -c ${ABSOLUTE_SRC} -o ${BC_FILE}
+ ${ARROW_GANDIVA_PC_CXX_FLAGS}
-I${CMAKE_SOURCE_DIR}/src
- DEPENDS ${SRC_FILE})
+ DEPENDS ${SRC_FILE}
+ COMMAND_EXPAND_LISTS)
list(APPEND BC_FILES ${BC_FILE})
endforeach()
diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/linux-test.sh
index 234ce8d..45efdb0 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/dev/tasks/python-wheels/linux-test.sh
@@ -24,10 +24,17 @@ pip install /arrow/python/manylinux1/dist/*.whl
python --version
# Test optional dependencies
-python -c "import pyarrow"
-python -c "import pyarrow.orc"
-python -c "import pyarrow.parquet"
-python -c "import pyarrow.plasma"
+command="
+import sys
+import pyarrow
+import pyarrow.orc
+import pyarrow.parquet
+import pyarrow.plasma
+
+if sys.version_info.major > 2:
+ import pyarrow.gandiva
+"
+python -c "$command"
# Run pyarrow tests
pip install -r /arrow/python/requirements-test.txt
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index a6e4123..3a048cc 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -61,6 +61,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
option(PYARROW_BUILD_CUDA
"Build the PyArrow CUDA support"
OFF)
+ option(PYARROW_BUILD_GANDIVA
+ "Build the PyArrow Gandiva integration"
+ OFF)
option(PYARROW_BUILD_PARQUET
"Build the PyArrow Parquet integration"
OFF)
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 902bcb3..7488d38 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -58,6 +58,15 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
PIP="${CPYTHON_PATH}/bin/pip"
PATH="$PATH:${CPYTHON_PATH}"
+ if [ $PYTHON != "2.7" ]; then
+ # Gandiva is not supported on Python 2.7
+ export PYARROW_WITH_GANDIVA=1
+ export BUILD_ARROW_GANDIVA=ON
+ else
+ export PYARROW_WITH_GANDIVA=0
+ export BUILD_ARROW_GANDIVA=OFF
+ fi
+
# TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7
if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then
if [ $PYTHON != "3.7" ]; then
@@ -75,6 +84,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
-DARROW_BUILD_TESTS=OFF \
-DARROW_BUILD_SHARED=ON \
-DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \
-DARROW_JEMALLOC=ON \
-DARROW_RPATH_ORIGIN=ON \
-DARROW_PYTHON=ON \
@@ -83,6 +93,8 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
-DARROW_PLASMA=ON \
-DARROW_TENSORFLOW=ON \
-DARROW_ORC=ON \
+ -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \
+ -DARROW_GANDIVA_JAVA=OFF \
-DBoost_NAMESPACE=arrow_boost \
-DBOOST_ROOT=/arrow_boost_dist \
-GNinja /arrow/cpp
@@ -98,6 +110,8 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
+ # Remove build directory to ensure CMake gets a clean run
+ rm -rf build/
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext \
--inplace \
--bundle-arrow-cpp \
@@ -114,6 +128,9 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
pip install repaired_wheels/*.whl
+ if [ $PYTHON != "2.7" ]; then
+ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.gandiva"
+ fi
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.orc"
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.parquet"
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER -c "import pyarrow.plasma"
diff --git a/python/setup.py b/python/setup.py
index 584c35a..99883bf 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -295,6 +295,8 @@ class build_ext(_build_ext):
move_shared_libs(build_prefix, build_lib, "arrow_gpu")
if self.with_plasma:
move_shared_libs(build_prefix, build_lib, "plasma")
+ if self.with_gandiva:
+ move_shared_libs(build_prefix, build_lib, "gandiva")
if self.with_parquet and not self.with_static_parquet:
move_shared_libs(build_prefix, build_lib, "parquet")
if not self.with_static_boost and self.bundle_boost: