You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/09/26 08:21:33 UTC
arrow git commit: ARROW-1606: [Python] Copy .lib files in addition to
.dll when bundling libraries for Windows
Repository: arrow
Updated Branches:
refs/heads/master b640cc0e7 -> 5da6b8795
ARROW-1606: [Python] Copy .lib files in addition to .dll when bundling libraries for Windows
I don't have access to a Windows machine at the moment to try this out locally. I will try from wesm/arrow-dist on Appveyor in the meantime
Author: Wes McKinney <we...@twosigma.com>
Closes #1127 from wesm/ARROW-1606 and squashes the following commits:
9edd85d [Wes McKinney] Bundle .lib files on Windows in CMakeLists.txt
e8f6c73 [Wes McKinney] Copy all suffixes for libraries on Windows to get .dll and .lib files
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/5da6b879
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/5da6b879
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/5da6b879
Branch: refs/heads/master
Commit: 5da6b87950c5bc6658bc49aef29dcf9bf67e0012
Parents: b640cc0
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Sep 26 10:21:28 2017 +0200
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Tue Sep 26 10:21:28 2017 +0200
----------------------------------------------------------------------
python/CMakeLists.txt | 83 +++++++++++++++++--------------
python/cmake_modules/FindArrow.cmake | 26 +++++++---
python/doc/source/development.rst | 3 +-
python/setup.py | 81 +++++++++++++++++-------------
4 files changed, 113 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index af95073..a636d51 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -176,6 +176,14 @@ find_package(Arrow REQUIRED)
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
function(bundle_arrow_lib library_path)
+ set(options)
+ set(one_value_args ABI_VERSION SO_VERSION)
+ set(multi_value_args)
+ cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+ if(ARG_UNPARSED_ARGUMENTS)
+ message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+ endif()
+
get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
configure_file(${${library_path}}
@@ -183,30 +191,45 @@ function(bundle_arrow_lib library_path)
COPYONLY)
if (APPLE)
- configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARG_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- else(NOT MSVC)
- configure_file(${${library_path}}.${ARROW_ABI_VERSION}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
+ elseif(NOT MSVC)
+ configure_file(${${library_path}}.${ARG_ABI_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_ABI_VERSION}
COPYONLY)
- configure_file(${${library_path}}.${ARROW_SO_VERSION}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}
+ configure_file(${${library_path}}.${ARG_SO_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARG_SO_VERSION}
COPYONLY)
endif()
- SET(ARROW_SHARED_LIB
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endfunction(bundle_arrow_lib)
+function(bundle_arrow_implib library_path)
+ get_filename_component(LIBRARY_DIR ${${library_path}} DIRECTORY)
+ get_filename_component(LIBRARY_NAME ${${library_path}} NAME_WE)
+ configure_file(${${library_path}}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.lib
+ COPYONLY)
+endfunction(bundle_arrow_implib)
+
if (PYARROW_BUNDLE_ARROW_CPP)
# arrow
file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
- bundle_arrow_lib(ARROW_SHARED_LIB)
- bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
+ bundle_arrow_lib(ARROW_SHARED_LIB
+ ABI_VERSION ${ARROW_ABI_VERSION}
+ SO_VERSION ${ARROW_SO_VERSION})
+ bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB
+ ABI_VERSION ${ARROW_ABI_VERSION}
+ SO_VERSION ${ARROW_SO_VERSION})
+
+ if (MSVC)
+ bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
+ bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
+ endif()
endif()
if (MSVC)
@@ -248,30 +271,12 @@ if (PYARROW_BUILD_PARQUET)
include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
if (PYARROW_BUNDLE_ARROW_CPP)
- get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
- get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
- message(STATUS "Configure start")
- configure_file(${PARQUET_SHARED_LIB}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
- COPYONLY)
- if (APPLE)
- configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- COPYONLY)
- configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
- COPYONLY)
- elseif (NOT MSVC)
- configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
- COPYONLY)
- configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
- COPYONLY)
- endif()
- message(STATUS "Configure end")
- #SET(PARQUET_SHARED_LIB
- # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+ bundle_arrow_lib(PARQUET_SHARED_LIB
+ ABI_VERSION ${PARQUET_ABI_VERSION}
+ SO_VERSION ${PARQUET_SO_VERSION})
+ if (MSVC)
+ bundle_arrow_implib(PARQUET_SHARED_IMP_LIB)
+ endif()
endif()
if (MSVC)
ADD_THIRDPARTY_LIB(parquet
@@ -301,7 +306,9 @@ if (PYARROW_BUILD_PLASMA)
SHARED_LIB ${PLASMA_SHARED_LIB})
if (PYARROW_BUNDLE_ARROW_CPP)
- bundle_arrow_lib(PLASMA_SHARED_LIB)
+ bundle_arrow_lib(PLASMA_SHARED_LIB
+ ABI_VERSION ${ARROW_ABI_VERSION}
+ SO_VERSION ${ARROW_SO_VERSION})
endif()
set(LINK_LIBS
${LINK_LIBS}
http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c893a1a..12f76b6 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -124,10 +124,22 @@ else ()
set(ARROW_FOUND FALSE)
endif ()
-mark_as_advanced(
- ARROW_INCLUDE_DIR
- ARROW_STATIC_LIB
- ARROW_SHARED_LIB
- ARROW_PYTHON_STATIC_LIB
- ARROW_PYTHON_SHARED_LIB
-)
+if (MSVC)
+ mark_as_advanced(
+ ARROW_INCLUDE_DIR
+ ARROW_STATIC_LIB
+ ARROW_SHARED_LIB
+ ARROW_SHARED_IMP_LIB
+ ARROW_PYTHON_STATIC_LIB
+ ARROW_PYTHON_SHARED_LIB
+ ARROW_PYTHON_SHARED_IMP_LIB
+ )
+else()
+ mark_as_advanced(
+ ARROW_INCLUDE_DIR
+ ARROW_STATIC_LIB
+ ARROW_SHARED_LIB
+ ARROW_PYTHON_STATIC_LIB
+ ARROW_PYTHON_SHARED_LIB
+ )
+endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/doc/source/development.rst
----------------------------------------------------------------------
diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst
index 53544ba..2d71323 100644
--- a/python/doc/source/development.rst
+++ b/python/doc/source/development.rst
@@ -272,7 +272,8 @@ Now, we build and install Arrow C++ libraries
cmake -G "Visual Studio 14 2015 Win64" ^
-DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
-DCMAKE_BUILD_TYPE=Release ^
- -DARROW_BUILD_TESTS=off ^
+ -DARROW_BUILD_TESTS=on ^
+ -DARROW_CXXFLAGS="/WX" ^
-DARROW_PYTHON=on ..
cmake --build . --target INSTALL --config Release
cd ..\..
http://git-wip-us.apache.org/repos/asf/arrow/blob/5da6b879/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 4657da0..8966fb8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -208,14 +208,6 @@ class build_ext(_build_ext):
# Move the libraries to the place expected by the Python
# build
- shared_library_prefix = 'lib'
- if sys.platform == 'darwin':
- shared_library_suffix = '.dylib'
- elif sys.platform == 'win32':
- shared_library_suffix = '.dll'
- shared_library_prefix = ''
- else:
- shared_library_suffix = '.so'
try:
os.makedirs(pjoin(build_lib, 'pyarrow'))
@@ -227,40 +219,18 @@ class build_ext(_build_ext):
else:
build_prefix = self.build_type
- def move_lib(lib_name):
- lib_filename = (shared_library_prefix + lib_name +
- shared_library_suffix)
- # Also copy libraries with ABI/SO version suffix
- if sys.platform == 'darwin':
- lib_pattern = (shared_library_prefix + lib_name +
- ".*" + shared_library_suffix[1:])
- libs = glob.glob(pjoin(build_prefix, lib_pattern))
- else:
- libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
- # Longest suffix library should be copied, all others symlinked
- libs.sort(key=lambda s: -len(s))
- print(libs, libs[0])
- lib_filename = os.path.basename(libs[0])
- shutil.move(pjoin(build_prefix, lib_filename),
- pjoin(build_lib, 'pyarrow', lib_filename))
- for lib in libs[1:]:
- filename = os.path.basename(lib)
- link_name = pjoin(build_lib, 'pyarrow', filename)
- if not os.path.exists(link_name):
- os.symlink(lib_filename, link_name)
-
if self.bundle_arrow_cpp:
print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
shutil.move(pjoin(build_prefix, 'include'),
pjoin(build_lib, 'pyarrow'))
- move_lib("arrow")
- move_lib("arrow_python")
+ move_shared_libs(build_prefix, build_lib, "arrow")
+ move_shared_libs(build_prefix, build_lib, "arrow_python")
if self.with_plasma:
- move_lib("plasma")
+ move_shared_libs(build_prefix, build_lib, "plasma")
if self.with_parquet:
- move_lib("parquet")
+ move_shared_libs(build_prefix, build_lib, "parquet")
# Move the built C-extension to the place expected by the Python build
self._found_names = []
@@ -344,6 +314,49 @@ class build_ext(_build_ext):
return [self._get_cmake_ext_path(name)
for name in self.get_names()]
+
+def move_shared_libs(build_prefix, build_lib, lib_name):
+ if sys.platform == 'win32':
+ # Move all .dll and .lib files
+ libs = glob.glob(pjoin(build_prefix, lib_name) + '*')
+
+ for filename in libs:
+ shutil.move(pjoin(build_prefix, filename),
+ pjoin(build_lib, 'pyarrow', filename))
+ else:
+ _move_shared_libs_unix(build_prefix, build_lib, lib_name)
+
+
+def _move_shared_libs_unix(build_prefix, build_lib, lib_name):
+ shared_library_prefix = 'lib'
+ if sys.platform == 'darwin':
+ shared_library_suffix = '.dylib'
+ else:
+ shared_library_suffix = '.so'
+
+ lib_filename = (shared_library_prefix + lib_name +
+ shared_library_suffix)
+ # Also copy libraries with ABI/SO version suffix
+ if sys.platform == 'darwin':
+ lib_pattern = (shared_library_prefix + lib_name +
+ ".*" + shared_library_suffix[1:])
+ libs = glob.glob(pjoin(build_prefix, lib_pattern))
+ else:
+ libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
+
+ # Longest suffix library should be copied, all others symlinked
+ libs.sort(key=lambda s: -len(s))
+ print(libs, libs[0])
+ lib_filename = os.path.basename(libs[0])
+ shutil.move(pjoin(build_prefix, lib_filename),
+ pjoin(build_lib, 'pyarrow', lib_filename))
+ for lib in libs[1:]:
+ filename = os.path.basename(lib)
+ link_name = pjoin(build_lib, 'pyarrow', filename)
+ if not os.path.exists(link_name):
+ os.symlink(lib_filename, link_name)
+
+
# In the case of a git-archive, we don't have any version information
# from the SCM to infer a version. The only source is the java/pom.xml.
#