You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/17 15:53:56 UTC
arrow git commit: ARROW-1030: Python: Account for library versioning
in parquet-cpp
Repository: arrow
Updated Branches:
refs/heads/master c4086febd -> a4f3259b0
ARROW-1030: Python: Account for library versioning in parquet-cpp
This mainly uses the same logic we already use for arrow-cpp
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #698 from xhochy/parquet-abi-version-bundling and squashes the following commits:
4aa17f82 [Uwe L. Korn] ARROW-1030: Python: Account for library versioning in parquet-cpp
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a4f3259b
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a4f3259b
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a4f3259b
Branch: refs/heads/master
Commit: a4f3259b07eeaa5300abf09cc46b867690c4d80f
Parents: c4086fe
Author: Uwe L. Korn <uw...@xhochy.com>
Authored: Wed May 17 11:53:51 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 17 11:53:51 2017 -0400
----------------------------------------------------------------------
python/CMakeLists.txt | 44 +++++++++++++++---
python/cmake_modules/FindParquet.cmake | 70 +++++++++++++++++++----------
python/manylinux1/Dockerfile-x86_64 | 2 +-
python/manylinux1/build_arrow.sh | 2 -
4 files changed, 85 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index ddd174c..ded85e8 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -290,16 +290,48 @@ if (PYARROW_BUILD_PARQUET)
message(FATAL_ERROR "Unable to locate Parquet libraries")
endif()
if (PYARROW_BUNDLE_ARROW_CPP)
+ get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY)
+ get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE)
+ get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE)
+ message(STATUS "Configure start")
configure_file(${PARQUET_SHARED_LIB}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- SET(PARQUET_SHARED_LIB
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
configure_file(${PARQUET_ARROW_SHARED_LIB}
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- SET(PARQUET_ARROW_SHARED_LIB
- ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+ if (APPLE)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ else()
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
+ COPYONLY)
+ configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION}
+ COPYONLY)
+ endif()
+ message(STATUS "Configure end")
+ #SET(PARQUET_SHARED_LIB
+ # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+ #SET(PARQUET_ARROW_SHARED_LIB
+ # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
ADD_THIRDPARTY_LIB(parquet_arrow
SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/cmake_modules/FindParquet.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index a20b651..ffd043d 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -20,35 +20,57 @@
# PARQUET_SHARED_LIB, path to libparquet's shared library
# PARQUET_FOUND, whether parquet has been found
-if( NOT "$ENV{PARQUET_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "$ENV{PARQUET_HOME}" _native_path )
- list( APPEND _parquet_roots ${_native_path} )
-elseif ( Parquet_HOME )
- list( APPEND _parquet_roots ${Parquet_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _parquet_roots )
- find_path( PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h
- PATHS ${_parquet_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "include" )
- find_library( PARQUET_LIBRARIES NAMES parquet
- PATHS ${_parquet_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib" )
+include(FindPkgConfig)
- find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
- PATHS ${_parquet_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib")
-else ()
- find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
- find_library(PARQUET_LIBRARIES NAMES parquet)
- find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
-endif ()
+if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
+ set(PARQUET_HOME "$ENV{PARQUET_HOME}")
+endif()
+if(PARQUET_HOME)
+ set(PARQUET_SEARCH_HEADER_PATHS
+ ${PARQUET_HOME}/include
+ )
+ set(PARQUET_SEARCH_LIB_PATH
+ ${PARQUET_HOME}/lib
+ )
+ find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS
+ ${PARQUET_SEARCH_HEADER_PATHS}
+ # make sure we don't accidentally pick up a different version
+ NO_DEFAULT_PATH
+ )
+ find_library(PARQUET_LIBRARIES NAMES parquet
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "lib")
+ find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "lib")
+ get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
+else()
+ pkg_check_modules(PARQUET parquet)
+ if (PARQUET_FOUND)
+ pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version)
+ message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}")
+ pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
+ message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}")
+ set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS})
+ set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS})
+ set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS})
+ message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}")
+ find_library(PARQUET_LIBRARIES NAMES parquet
+ PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
+ find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow
+ PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
+ message(STATUS "${PARQUET_ARROW_LIBRARIES}")
+ else()
+ find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
+ find_library(PARQUET_LIBRARIES NAMES parquet)
+ find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow)
+ get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
+ endif()
+endif()
if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
set(PARQUET_FOUND TRUE)
- get_filename_component( PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
set(PARQUET_LIB_NAME libparquet)
set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/manylinux1/Dockerfile-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index ad40e79..363308b 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -21,5 +21,5 @@ RUN make -j5 install
WORKDIR /
RUN git clone https://github.com/apache/parquet-cpp.git
WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF .
RUN make -j5 install
http://git-wip-us.apache.org/repos/asf/arrow/blob/a4f3259b/python/manylinux1/build_arrow.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index e072749..4565e59 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -39,9 +39,7 @@ export PYARROW_WITH_JEMALLOC=1
export PYARROW_BUNDLE_ARROW_CPP=1
# Need as otherwise arrow_io is sometimes not linked
export LDFLAGS="-Wl,--no-as-needed"
-export PARQUET_HOME="/usr"
export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
-
# Ensure the target directory exists
mkdir -p /io/dist