You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/23 18:09:40 UTC
arrow git commit: ARROW-1034: [PYTHON] Resolve wheel build issues on
Windows
Repository: arrow
Updated Branches:
refs/heads/master 1863a0193 -> bc70fae14
ARROW-1034: [PYTHON] Resolve wheel build issues on Windows
Author: Max Risuhin <ri...@gmail.com>
Closes #707 from MaxRis/ARROW-1034 and squashes the following commits:
796d1554 [Max Risuhin] ARROW-1034: [PYTHON] Resolve wheel build issues on Windows
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bc70fae1
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bc70fae1
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bc70fae1
Branch: refs/heads/master
Commit: bc70fae14e01bd2c05776b6497fe4eacfeb6a8a0
Parents: 1863a01
Author: Max Risuhin <ri...@gmail.com>
Authored: Tue May 23 14:09:36 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue May 23 14:09:36 2017 -0400
----------------------------------------------------------------------
ci/msvc-build.bat | 2 +-
python/CMakeLists.txt | 34 +++++++++++++++++-------
python/cmake_modules/FindArrow.cmake | 41 ++++++++++++++++++++++-------
python/cmake_modules/FindParquet.cmake | 41 ++++++++++++++++++++++-------
python/setup.py | 23 +++++++++++-----
5 files changed, 105 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc70fae1/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index f756fc5..d13c11f 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -68,6 +68,6 @@ popd
set PYTHONPATH=
pushd python
-python setup.py build_ext --inplace --with-parquet || exit /B
+python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel || exit /B
py.test pyarrow -v -s || exit /B
popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc70fae1/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6f48f7f..b02f23b 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -237,7 +237,7 @@ function(bundle_arrow_lib library_path)
configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- else()
+ else(NOT MSVC)
configure_file(${${library_path}}.${ARROW_ABI_VERSION}
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
COPYONLY)
@@ -257,10 +257,17 @@ if (PYARROW_BUNDLE_ARROW_CPP)
bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
endif()
-ADD_THIRDPARTY_LIB(arrow
- SHARED_LIB ${ARROW_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_python
- SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+if (MSVC)
+ ADD_THIRDPARTY_LIB(arrow
+ SHARED_LIB ${ARROW_SHARED_IMP_LIB})
+ ADD_THIRDPARTY_LIB(arrow_python
+ SHARED_LIB ${ARROW_PYTHON_SHARED_IMP_LIB})
+else()
+ ADD_THIRDPARTY_LIB(arrow
+ SHARED_LIB ${ARROW_SHARED_LIB})
+ ADD_THIRDPARTY_LIB(arrow_python
+ SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+endif()
############################################################
# Subdirectories
@@ -312,7 +319,7 @@ if (PYARROW_BUILD_PARQUET)
configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- else()
+ elseif (NOT MSVC)
configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
COPYONLY)
@@ -332,10 +339,17 @@ if (PYARROW_BUILD_PARQUET)
#SET(PARQUET_ARROW_SHARED_LIB
# ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
- ADD_THIRDPARTY_LIB(parquet
- SHARED_LIB ${PARQUET_SHARED_LIB})
- ADD_THIRDPARTY_LIB(parquet_arrow
- SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+ if (MSVC)
+ ADD_THIRDPARTY_LIB(parquet
+ SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
+ ADD_THIRDPARTY_LIB(parquet_arrow
+ SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB})
+ else()
+ ADD_THIRDPARTY_LIB(parquet
+ SHARED_LIB ${PARQUET_SHARED_LIB})
+ ADD_THIRDPARTY_LIB(parquet_arrow
+ SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+ endif()
set(LINK_LIBS
${LINK_LIBS}
parquet_shared
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc70fae1/python/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c16a4bf..4c8ed3d 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -21,6 +21,7 @@
# ARROW_LIBS, directory containing arrow libraries
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_SHARED_LIB, path to libarrow's shared library
+# ARROW_SHARED_IMP_LIB, path to libarrow's import library (MSVC only)
# ARROW_FOUND, whether arrow has been found
include(FindPkgConfig)
@@ -69,23 +70,45 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
PATHS
${ARROW_SEARCH_LIB_PATH}
NO_DEFAULT_PATH)
+get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)
+
+if (MSVC)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+ if (MSVC AND NOT ARROW_MSVC_STATIC_LIB_SUFFIX)
+ set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
+ endif()
+
+ find_library(ARROW_SHARED_LIBRARIES NAMES arrow
+ PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+
+ find_library(ARROW_PYTHON_SHARED_LIBRARIES NAMES arrow_python
+ PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+ get_filename_component(ARROW_SHARED_LIBS ${ARROW_SHARED_LIBRARIES} PATH )
+ get_filename_component(ARROW_PYTHON_SHARED_LIBS ${ARROW_PYTHON_SHARED_LIBRARIES} PATH )
+endif ()
if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
set(ARROW_FOUND TRUE)
-
+ set(ARROW_LIB_NAME arrow)
+ set(ARROW_PYTHON_LIB_NAME arrow_python)
if (MSVC)
- set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
- set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
- set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
- set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+ set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_LIB ${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
+ set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
else()
- set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
- set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+ set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
+ set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
- set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
- set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc70fae1/python/cmake_modules/FindParquet.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index de53a29..ef3c645 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -21,6 +21,7 @@
# PARQUET_LIBS, directory containing parquet libraries
# PARQUET_STATIC_LIB, path to libparquet.a
# PARQUET_SHARED_LIB, path to libparquet's shared library
+# PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only)
# PARQUET_FOUND, whether parquet has been found
include(FindPkgConfig)
@@ -29,6 +30,24 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
set(PARQUET_HOME "$ENV{PARQUET_HOME}")
endif()
+if (MSVC)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+ if (MSVC AND NOT PARQUET_MSVC_STATIC_LIB_SUFFIX)
+ set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
+ endif()
+
+ find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+
+ find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+ get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
+ get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH )
+endif ()
+
if(PARQUET_HOME)
set(PARQUET_SEARCH_HEADER_PATHS
${PARQUET_HOME}/include
@@ -74,13 +93,14 @@ endif()
if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
set(PARQUET_FOUND TRUE)
+ set(PARQUET_LIB_NAME parquet)
if (MSVC)
- set(PARQUET_STATIC_LIB "${PARQUET_LIBRARIES}_static")
- set(PARQUET_SHARED_LIB "${PARQUET_LIBRARIES}")
+ set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib")
else()
- set(PARQUET_LIB_NAME libparquet)
- set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
- set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a)
+ set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
else ()
set(PARQUET_FOUND FALSE)
@@ -89,15 +109,16 @@ endif ()
if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
set(PARQUET_ARROW_FOUND TRUE)
get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
+ set(PARQUET_ARROW_LIB_NAME parquet_arrow)
if (MSVC)
- set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBRARIES}_static")
- set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_LIBRARIES}")
+ set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib")
else()
- set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
set(PARQUET_ARROW_STATIC_LIB
- ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+ ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a)
set(PARQUET_ARROW_SHARED_LIB
- ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
else ()
set(PARQUET_ARROW_FOUND FALSE)
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc70fae1/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 9374af3..7e612c9 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -29,7 +29,7 @@ import Cython
import pkg_resources
-from setuptools import setup, Extension
+from setuptools import setup, Extension, Distribution
from os.path import join as pjoin
@@ -210,6 +210,11 @@ class build_ext(_build_ext):
except OSError:
pass
+ if sys.platform == 'win32':
+ build_prefix = ''
+ else:
+ build_prefix = self.build_type
+
def move_lib(lib_name):
lib_filename = (shared_library_prefix + lib_name +
shared_library_suffix)
@@ -217,14 +222,14 @@ class build_ext(_build_ext):
if sys.platform == 'darwin':
lib_pattern = (shared_library_prefix + lib_name +
".*" + shared_library_suffix[1:])
- libs = glob.glob(pjoin(self.build_type, lib_pattern))
+ libs = glob.glob(pjoin(build_prefix, lib_pattern))
else:
- libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+ libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
# Longest suffix library should be copied, all others symlinked
libs.sort(key=lambda s: -len(s))
print(libs, libs[0])
lib_filename = os.path.basename(libs[0])
- shutil.move(pjoin(self.build_type, lib_filename),
+ shutil.move(pjoin(build_prefix, lib_filename),
pjoin(build_lib, 'pyarrow', lib_filename))
for lib in libs[1:]:
filename = os.path.basename(lib)
@@ -233,10 +238,10 @@ class build_ext(_build_ext):
os.symlink(lib_filename, link_name)
if self.bundle_arrow_cpp:
- print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+ print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
- shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+ shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
move_lib("arrow")
move_lib("arrow_python")
if self.with_jemalloc:
@@ -337,11 +342,17 @@ representations of flat and hierarchical data along with multiple
language-bindings for structure manipulation. It also provides IPC
and common algorithm implementations."""
+class BinaryDistribution(Distribution):
+ def has_ext_modules(foo):
+ return True
+
setup(
name="pyarrow",
packages=['pyarrow', 'pyarrow.tests'],
zip_safe=False,
package_data={'pyarrow': ['*.pxd', '*.pyx']},
+ include_package_data=True,
+ distclass=BinaryDistribution,
# Dummy extension to trigger build_ext
ext_modules=[Extension('__dummy__', sources=[])],