You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/02/08 15:09:34 UTC
[arrow] branch master updated: ARROW-4446: [C++][Python] Run
Gandiva C++ unit tests in Appveyor, get build and tests working in Python
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 29f76df ARROW-4446: [C++][Python] Run Gandiva C++ unit tests in Appveyor, get build and tests working in Python
29f76df is described below
commit 29f76df0a90927d1e8fab7f70852448b1093a151
Author: Wes McKinney <we...@apache.org>
AuthorDate: Fri Feb 8 09:09:25 2019 -0600
ARROW-4446: [C++][Python] Run Gandiva C++ unit tests in Appveyor, get build and tests working in Python
Resolves
* ARROW-4014 Fix warning about LIBCMT in MSVC build
* ARROW-4445 Run Gandiva C++ tests in Appveyor
* ARROW-4446 Run Gandiva Python unit tests in Appveyor
* ARROW-4404 Appveyor toolchain build does not actually build the project
Author: Wes McKinney <we...@apache.org>
Closes #3567 from wesm/gandiva-python-windows and squashes the following commits:
d9e4b1bef <Wes McKinney> Only use /nodefaultlib:libcmt if arrow_use_static_crt=OFF
a8f960d07 <Wes McKinney> Fix appyveyor.yml
72c97e26a <Wes McKinney> Re-enable all Appyveyor entries
a2e57d1e0 <Wes McKinney> Prevent move_shared_libs from clobbering the gandiva.cpp file which downstream code is expecting to find
d78ae1336 <Wes McKinney> Debug prints
ac402f0fc <Wes McKinney> Set language_level = 3
8ae63f199 <Wes McKinney> Fix LLVM 7 issues with MSVC, enable FindClangTools.cmake to find clang-format in conda env
3677115f4 <Wes McKinney> Do not do verbose linking
773c7319c <Wes McKinney> Fixes
00d412676 <Wes McKinney> Disable failing Windows tests
063543cad <Wes McKinney> Don't touch linker flags unless on MSVC
e19b68876 <Wes McKinney> Upgrade to LLVM 7
707bc492a <Wes McKinney> Fix for gflags
5897045e1 <Wes McKinney> Retool FindGFlags.cmake a bit to try to fix Windows toolchain build
c2bb42080 <Wes McKinney> Install all conda toolchain packages in one go
c2857762b <Wes McKinney> Rearrange conda calls to see if fixes build
469a2dff9 <Wes McKinney> Enable gandiva is one build entry
b05cf0814 <Wes McKinney> Get Python build and tests for Gandiva working on, add to Appveyor build
---
appveyor.yml | 2 +
ci/appveyor-cpp-build.bat | 31 +++++---------
ci/cpp-msvc-build-main.bat | 4 +-
cpp/CMakeLists.txt | 4 ++
cpp/cmake_modules/FindClangTools.cmake | 6 ++-
cpp/cmake_modules/FindGFlags.cmake | 23 ++++++----
cpp/cmake_modules/FindGandiva.cmake | 30 ++++++++-----
cpp/cmake_modules/FindParquet.cmake | 7 +++-
cpp/cmake_modules/SetupCxxFlags.cmake | 19 +++++++++
cpp/src/gandiva/engine.cc | 6 +++
cpp/src/gandiva/expression_registry.h | 1 +
cpp/src/gandiva/precompiled/time_test.cc | 16 +++++++
docs/source/python/development.rst | 2 +
python/CMakeLists.txt | 15 ++++++-
python/pyarrow/gandiva.pyx | 1 +
python/setup.py | 72 +++++++++++++++++---------------
16 files changed, 163 insertions(+), 76 deletions(-)
diff --git a/appveyor.yml b/appveyor.yml
index d955484..e529ef7 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -48,6 +48,7 @@ environment:
global:
USE_CLCACHE: true
ARROW_BUILD_GANDIVA: "OFF"
+ ARROW_LLVM_VERSION: "7.0.*"
PYTHON: "3.6"
ARCH: "64"
@@ -61,6 +62,7 @@ environment:
- JOB: "Toolchain"
GENERATOR: Visual Studio 14 2015 Win64
CONFIGURATION: "Release"
+ ARROW_BUILD_GANDIVA: "ON"
- JOB: "Static_Crt_Build"
GENERATOR: Ninja
- JOB: "Build_Debug"
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index f95b88e..21ee6a5 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -95,33 +95,24 @@ if "%JOB%" == "Build_Debug" (
exit /B 0
)
-conda create -n arrow -q -y -c conda-forge ^
- --file=ci\conda_env_python.yml ^
- python=%PYTHON% ^
- numpy=1.14 ^
- thrift-cpp=0.11 ^
- boost-cpp
-
-call activate arrow
-
-set ARROW_LLVM_VERSION=6.0.1
+set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14 thrift-cpp=0.11 boost-cpp
if "%ARROW_BUILD_GANDIVA%" == "ON" (
@rem Install llvmdev in the toolchain if building gandiva.dll
- conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B
+ set CONDA_PACKAGES=%CONDA_PACKAGES% llvmdev=%ARROW_LLVM_VERSION% clangdev=%ARROW_LLVM_VERSION%
)
-@rem Use Boost from Anaconda
-set BOOST_ROOT=%CONDA_PREFIX%\Library
-set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
-
if "%JOB%" == "Toolchain" (
@rem Install pre-built "toolchain" packages for faster builds
- conda install -q -y -c conda-forge ^
- --file=ci\conda_env_cpp.yml ^
- python=%PYTHON%
-
- set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml
)
+conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge
+
+call activate arrow
+
+@rem Use Boost from Anaconda
+set BOOST_ROOT=%CONDA_PREFIX%\Library
+set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
+
call ci\cpp-msvc-build-main.bat
diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat
index 5b1842d..db15de6 100644
--- a/ci/cpp-msvc-build-main.bat
+++ b/ci/cpp-msvc-build-main.bat
@@ -22,7 +22,8 @@ set ARROW_HOME=%CONDA_PREFIX%\Library
set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF
if "%JOB%" == "Toolchain" (
- set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
+ set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
+ set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
)
@rem Retrieve git submodules, configure env var for Parquet unit tests
@@ -85,6 +86,7 @@ set PYARROW_BUNDLE_ARROW_CPP=ON
set PYARROW_BUNDLE_BOOST=OFF
set PYARROW_WITH_STATIC_BOOST=ON
set PYARROW_WITH_PARQUET=ON
+set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
set PYARROW_PARALLEL=2
@rem ARROW-3075; pkgconfig is broken for Parquet for now
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index b81ba64..bf02d1a 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -311,6 +311,10 @@ Note that this requires linking Boost statically"
# Windows options
if (MSVC)
+ option(MSVC_LINK_VERBOSE
+ "Pass verbose linking options when linking libraries and executables"
+ OFF)
+
option(ARROW_USE_CLCACHE
"Use clcache if available"
ON)
diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake
index f8d76a7..e7137f9 100644
--- a/cpp/cmake_modules/FindClangTools.cmake
+++ b/cpp/cmake_modules/FindClangTools.cmake
@@ -53,8 +53,10 @@ set(CLANG_TOOLS_SEARCH_PATHS
${ClangTools_PATH}
$ENV{CLANG_TOOLS_PATH}
/usr/local/bin /usr/bin
- "C:/Program Files/LLVM/bin"
- "${HOMEBREW_PREFIX}/bin")
+ "C:/Program Files/LLVM/bin" # Windows, non-conda
+ "$ENV{CONDA_PREFIX}/Library/bin" # Windows, conda
+ "${HOMEBREW_PREFIX}/bin"
+)
find_program(CLANG_TIDY_BIN
NAMES clang-tidy-${ARROW_LLVM_VERSION}
diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
index f71950c..cdf6dae 100644
--- a/cpp/cmake_modules/FindGFlags.cmake
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -39,16 +39,18 @@ set(GFLAGS_STATIC_LIB_SUFFIX
set(GFLAGS_STATIC_LIB_NAME
${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_STATIC_LIB_SUFFIX})
+message(STATUS "GFLAGS_HOME: ${GFLAGS_HOME}")
+
if ( _gflags_roots )
find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "include" )
- find_library(GFLAGS_SHARED_LIB NAMES gflags
+ find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
- find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
+ find_library(GFLAGS_SHARED_LIB NAMES gflags
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
@@ -57,14 +59,21 @@ else()
# make sure we don't accidentally pick up a different version
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
- find_library(GFLAGS_SHARED_LIB gflags
+ find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
- find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
+ find_library(GFLAGS_SHARED_LIB gflags
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
endif()
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(GFLAGS REQUIRED_VARS
- GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
+if (GFLAGS_INCLUDE_DIR AND GFLAGS_STATIC_LIB)
+ set(GFLAGS_FOUND TRUE)
+else ()
+ set(GFLAGS_FOUND FALSE)
+endif ()
+
+mark_as_advanced(
+ GFLAGS_INCLUDE_DIR
+ GFLAGS_STATIC_LIB
+)
diff --git a/cpp/cmake_modules/FindGandiva.cmake b/cpp/cmake_modules/FindGandiva.cmake
index 5559c09..81b1206 100644
--- a/cpp/cmake_modules/FindGandiva.cmake
+++ b/cpp/cmake_modules/FindGandiva.cmake
@@ -45,9 +45,7 @@ else()
${GANDIVA_HOME}/include
)
- set(GANDIVA_SEARCH_LIB_PATH
- ${GANDIVA_HOME}/lib
- )
+ set(GANDIVA_SEARCH_LIB_PATH "${GANDIVA_HOME}")
find_path(GANDIVA_INCLUDE_DIR gandiva/expression_registry.h PATHS
${GANDIVA_SEARCH_HEADER_PATHS}
@@ -57,18 +55,30 @@ else()
endif()
find_library(GANDIVA_LIB_PATH NAMES gandiva
- PATHS
- ${GANDIVA_SEARCH_LIB_PATH}
- NO_DEFAULT_PATH)
+ PATHS ${GANDIVA_SEARCH_LIB_PATH}
+ NO_DEFAULT_PATH
+ PATH_SUFFIXES "lib")
get_filename_component(GANDIVA_LIBS ${GANDIVA_LIB_PATH} DIRECTORY)
-if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS)
+find_library(GANDIVA_SHARED_LIB_PATH NAMES gandiva
+ PATHS ${GANDIVA_SEARCH_LIB_PATH}
+ NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+
+get_filename_component(GANDIVA_SHARED_LIBS ${GANDIVA_SHARED_LIB_PATH} PATH )
+
+if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS AND GANDIVA_SHARED_LIBS)
set(GANDIVA_FOUND TRUE)
set(GANDIVA_LIB_NAME gandiva)
- set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}.a)
-
- set(GANDIVA_SHARED_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ if (MSVC)
+ set(GANDIVA_STATIC_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}${GANDIVA_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(GANDIVA_SHARED_LIB "${GANDIVA_SHARED_LIBS}/${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(GANDIVA_SHARED_IMP_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}.lib")
+ else()
+ set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+ set(GANDIVA_SHARED_LIB ${GANDIVA_SHARED_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ endif()
endif()
if (GANDIVA_FOUND)
diff --git a/cpp/cmake_modules/FindParquet.cmake b/cpp/cmake_modules/FindParquet.cmake
index 68b004c..1a3e229 100644
--- a/cpp/cmake_modules/FindParquet.cmake
+++ b/cpp/cmake_modules/FindParquet.cmake
@@ -30,6 +30,10 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
set(PARQUET_HOME "$ENV{PARQUET_HOME}")
endif()
+if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
+ set(ARROW_HOME "$ENV{ARROW_HOME}")
+endif()
+
if (MSVC)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
@@ -38,7 +42,8 @@ if (MSVC)
endif()
find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
- PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATHS ${PARQUET_HOME} ${ARROW_HOME}
+ NO_DEFAULT_PATH
PATH_SUFFIXES "bin" )
get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index cb64ea5..c71bf15 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -411,3 +411,22 @@ else()
endif()
message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
+
+# ----------------------------------------------------------------------
+# MSVC-specific linker options
+
+if (MSVC)
+ set(MSVC_LINKER_FLAGS)
+ if (MSVC_LINK_VERBOSE)
+ set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /VERBOSE:LIB")
+ endif()
+ if (NOT ARROW_USE_STATIC_CRT)
+ set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /NODEFAULTLIB:LIBCMT")
+ set(CMAKE_EXE_LINKER_FLAGS
+ "${CMAKE_EXE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+ set(CMAKE_MODULE_LINKER_FLAGS
+ "${CMAKE_MODULE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+ set(CMAKE_SHARED_LINKER_FLAGS
+ "${CMAKE_SHARED_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+ endif()
+endif()
diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc
index 113ba81..c816ec7 100644
--- a/cpp/src/gandiva/engine.cc
+++ b/cpp/src/gandiva/engine.cc
@@ -15,6 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+// TODO(wesm): LLVM 7 produces pesky C4244 that disable pragmas around the LLVM
+// includes seem to not fix as with LLVM 6
+#if defined(_MSC_VER)
+#pragma warning(disable : 4244)
+#endif
+
#include "gandiva/engine.h"
#include <iostream>
diff --git a/cpp/src/gandiva/expression_registry.h b/cpp/src/gandiva/expression_registry.h
index 4524a07..97197f2 100644
--- a/cpp/src/gandiva/expression_registry.h
+++ b/cpp/src/gandiva/expression_registry.h
@@ -63,6 +63,7 @@ class GANDIVA_EXPORT ExpressionRegistry {
std::unique_ptr<FunctionRegistry> function_registry_;
};
+GANDIVA_EXPORT
std::vector<std::shared_ptr<FunctionSignature>> GetRegisteredFunctionSignatures();
} // namespace gandiva
diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc
index b8f8069..6f337d6 100644
--- a/cpp/src/gandiva/precompiled/time_test.cc
+++ b/cpp/src/gandiva/precompiled/time_test.cc
@@ -67,6 +67,15 @@ TEST(TestTime, TestCastTimestamp) {
-1187308799080);
EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1857-02-11 20:31:40.920 -05:30", 30),
-3562264699080);
+}
+
+#ifndef _WIN32
+
+// TODO(wesm): ARROW-4495. Need to address TZ database issues on Windows
+
+TEST(TestTime, TestCastTimestampWithTZ) {
+ ExecutionContext context;
+ int64_t context_ptr = reinterpret_cast<int64_t>(&context);
EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 Canada/Pacific", 37),
969727530920);
@@ -74,6 +83,11 @@ TEST(TestTime, TestCastTimestamp) {
1330452059000);
EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1923-10-07 03:03:03 America/New_York", 36),
-1459094217000);
+}
+
+TEST(TestTime, TestCastTimestampErrors) {
+ ExecutionContext context;
+ int64_t context_ptr = reinterpret_cast<int64_t>(&context);
// error cases
EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "20000923", 8), 0);
@@ -93,6 +107,8 @@ TEST(TestTime, TestCastTimestamp) {
context.Reset();
}
+#endif
+
TEST(TestTime, TestExtractTime) {
// 10:20:33
int32 time_as_millis_in_day = 37233000;
diff --git a/docs/source/python/development.rst b/docs/source/python/development.rst
index d855371..7a9e8cb 100644
--- a/docs/source/python/development.rst
+++ b/docs/source/python/development.rst
@@ -325,6 +325,8 @@ Now, we build and install Arrow C++ libraries
cmake --build . --target INSTALL --config Release
cd ..\..
+If building with LLVM, also add `-DARROW_GANDIVA=ON`.
+
After that, we must put the install directory's bin path in our ``%PATH%``:
.. code-block:: shell
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 35f1b82..ad27ea3 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -517,12 +517,23 @@ if (PYARROW_BUILD_GANDIVA)
bundle_arrow_lib(GANDIVA_SHARED_LIB
ABI_VERSION ${ARROW_ABI_VERSION}
SO_VERSION ${ARROW_SO_VERSION})
+
+ if (MSVC)
+ bundle_arrow_implib(GANDIVA_SHARED_IMP_LIB)
+ endif()
+ endif()
+
+ if (MSVC)
+ ADD_THIRDPARTY_LIB(gandiva
+ SHARED_LIB ${GANDIVA_SHARED_IMP_LIB})
+ else()
+ ADD_THIRDPARTY_LIB(gandiva
+ SHARED_LIB ${GANDIVA_SHARED_LIB})
endif()
set(LINK_LIBS
${LINK_LIBS}
- ${GANDIVA_SHARED_LIB})
-
+ gandiva_shared)
set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} gandiva)
endif()
diff --git a/python/pyarrow/gandiva.pyx b/python/pyarrow/gandiva.pyx
index 76e55d6..3904a8a 100644
--- a/python/pyarrow/gandiva.pyx
+++ b/python/pyarrow/gandiva.pyx
@@ -18,6 +18,7 @@
# cython: profile=False
# distutils: language = c++
# cython: embedsignature = True
+# cython: language_level = 3
from libcpp cimport bool as c_bool, nullptr
from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
diff --git a/python/setup.py b/python/setup.py
index 6767300..b10f4e3 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -158,13 +158,13 @@ class build_ext(_build_ext):
os.environ.get('PYARROW_BUNDLE_BOOST', '0'))
CYTHON_MODULE_NAMES = [
- 'gandiva',
'lib',
'_csv',
'_cuda',
'_parquet',
'_orc',
- '_plasma']
+ '_plasma',
+ 'gandiva']
def _run_cmake(self):
# The directory containing this setup.py
@@ -287,33 +287,6 @@ class build_ext(_build_ext):
else:
build_prefix = self.build_type
- if self.bundle_arrow_cpp:
- print(pjoin(build_lib, 'pyarrow'))
- move_shared_libs(build_prefix, build_lib, "arrow")
- move_shared_libs(build_prefix, build_lib, "arrow_python")
- if self.with_cuda:
- move_shared_libs(build_prefix, build_lib, "arrow_gpu")
- if self.with_plasma:
- move_shared_libs(build_prefix, build_lib, "plasma")
- if self.with_gandiva:
- move_shared_libs(build_prefix, build_lib, "gandiva")
- if self.with_parquet and not self.with_static_parquet:
- move_shared_libs(build_prefix, build_lib, "parquet")
- if not self.with_static_boost and self.bundle_boost:
- move_shared_libs(
- build_prefix, build_lib,
- "{}_filesystem".format(self.boost_namespace))
- move_shared_libs(
- build_prefix, build_lib,
- "{}_system".format(self.boost_namespace))
- move_shared_libs(
- build_prefix, build_lib,
- "{}_regex".format(self.boost_namespace))
- if sys.platform == 'win32':
- # zlib uses zlib.dll for Windows
- zlib_lib_name = 'zlib'
- move_shared_libs(build_prefix, build_lib, zlib_lib_name)
-
print('Bundling includes: ' + pjoin(build_prefix, 'include'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
@@ -326,7 +299,7 @@ class build_ext(_build_ext):
for name in self.CYTHON_MODULE_NAMES:
built_path = self.get_ext_built(name)
if not os.path.exists(built_path):
- print(built_path)
+ print('Did not find {0}'.format(built_path))
if self._failure_permitted(name):
print('Cython module {0} failure permitted'
.format(name))
@@ -365,6 +338,37 @@ class build_ext(_build_ext):
pjoin(os.path.dirname(ext_path),
name + '_api.h'))
+ if self.bundle_arrow_cpp:
+ print(pjoin(build_lib, 'pyarrow'))
+ move_shared_libs(build_prefix, build_lib, "arrow")
+ move_shared_libs(build_prefix, build_lib, "arrow_python")
+ if self.with_cuda:
+ move_shared_libs(build_prefix, build_lib, "arrow_gpu")
+ if self.with_plasma:
+ move_shared_libs(build_prefix, build_lib, "plasma")
+ if self.with_gandiva:
+ move_shared_libs(build_prefix, build_lib, "gandiva")
+ if self.with_parquet and not self.with_static_parquet:
+ move_shared_libs(build_prefix, build_lib, "parquet")
+ if not self.with_static_boost and self.bundle_boost:
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_filesystem".format(self.boost_namespace),
+ implib_required=False)
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_system".format(self.boost_namespace),
+ implib_required=False)
+ move_shared_libs(
+ build_prefix, build_lib,
+ "{}_regex".format(self.boost_namespace),
+ implib_required=False)
+ if sys.platform == 'win32':
+ # zlib uses zlib.dll for Windows
+ zlib_lib_name = 'zlib'
+ move_shared_libs(build_prefix, build_lib, zlib_lib_name,
+ implib_required=False)
+
if self.with_plasma:
# Move the plasma store
source = os.path.join(self.build_type, "plasma_store_server")
@@ -437,11 +441,13 @@ class build_ext(_build_ext):
for name in self.get_names()]
-def move_shared_libs(build_prefix, build_lib, lib_name):
+def move_shared_libs(build_prefix, build_lib, lib_name,
+ implib_required=True):
if sys.platform == 'win32':
# Move all .dll and .lib files
- libs = glob.glob(pjoin(build_prefix, lib_name) + '*')
-
+ libs = [lib_name + '.dll']
+ if implib_required:
+ libs.append(lib_name + '.lib')
for filename in libs:
shutil.move(pjoin(build_prefix, filename),
pjoin(build_lib, 'pyarrow', filename))