You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/01/18 00:09:31 UTC
arrow git commit: ARROW-490: Python: Update manylinux1 build scripts
Repository: arrow
Updated Branches:
refs/heads/master 850774efe -> b1472305c
ARROW-490: Python: Update manylinux1 build scripts
Through the usage of the ExternalProject command, a lot has become much
simpler.
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #290 from xhochy/ARROW-490 and squashes the following commits:
64b024b [Uwe L. Korn] ARROW-490: Python: Update manylinux1 build scripts
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b1472305
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b1472305
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b1472305
Branch: refs/heads/master
Commit: b1472305c7262b07f2e9bf3d5db4eeff09ec5dc4
Parents: 850774e
Author: Uwe L. Korn <uw...@xhochy.com>
Authored: Tue Jan 17 19:09:24 2017 -0500
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Jan 17 19:09:24 2017 -0500
----------------------------------------------------------------------
cpp/CMakeLists.txt | 4 ++--
cpp/cmake_modules/BuildUtils.cmake | 12 ++++++++++
.../Dockerfile-parquet_arrow-base-x86_64 | 23 +-------------------
python/manylinux1/Dockerfile-x86_64 | 19 +++++-----------
python/manylinux1/build_arrow.sh | 8 ++++---
5 files changed, 25 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8a2cfc5..885ab19 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -598,7 +598,7 @@ if (ARROW_JEMALLOC)
find_package(jemalloc)
if(NOT JEMALLOC_FOUND)
- set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep")
+ set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/")
set(JEMALLOC_HOME "${JEMALLOC_PREFIX}")
set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include")
set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -611,7 +611,7 @@ if (ARROW_JEMALLOC)
CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix="
BUILD_IN_SOURCE 1
BUILD_COMMAND ${MAKE}
- BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}")
+ BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}")
else()
ExternalProject_Add(jemalloc_ep
URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2
http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/cpp/cmake_modules/BuildUtils.cmake
----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index b620de5..9de9de5 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -27,6 +27,18 @@ function(ADD_ARROW_LIB LIB_NAME)
add_library(${LIB_NAME}_objlib OBJECT
${ARG_SOURCES}
)
+ if (ARG_STATIC_LINK_LIBS)
+ add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_LINK_LIBS})
+ endif()
+ if (ARG_STATIC_PRIVATE_LINK_LIBS)
+ add_dependencies(${LIB_NAME}_objlib ${ARG_STATIC_PRIVATE_LINK_LIBS})
+ endif()
+ if (ARG_SHARED_LINK_LIBS)
+ add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_LINK_LIBS})
+ endif()
+ if(ARG_SHARED_PRIVATE_LINK_LIBS)
+ add_dependencies(${LIB_NAME}_objlib ${ARG_SHARED_PRIVATE_LINK_LIBS})
+ endif()
# Necessary to make static linking into other shared libraries work properly
set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64 b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
index 714fa1a..94f5bc0 100644
--- a/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
+++ b/python/manylinux1/Dockerfile-parquet_arrow-base-x86_64
@@ -13,28 +13,7 @@
FROM arrow-base-x86_64
WORKDIR /
-ADD http://zlib.net/zlib-1.2.8.tar.gz /zlib-1.2.8.tar.gz
-RUN tar xf zlib-1.2.8.tar.gz
-WORKDIR zlib-1.2.8
-RUN CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DCMAKE_BUILD_TYPE=Release .
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz /snappy-1.1.3.tar.gz
-RUN tar xf snappy-1.1.3.tar.gz
-WORKDIR /snappy-1.1.3
-RUN ./configure --with-pic --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz /thrift-0.9.1.tar.gz
-RUN tar xf thrift-0.9.1.tar.gz
-WORKDIR /thrift-0.9.1
-RUN ./configure LDFLAGS='-L/usr/lib64' CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --without-tests --with-cpp --prefix=/usr --disable-shared --enable-static
-RUN make -j5 install
-
-WORKDIR /
RUN git clone https://github.com/apache/parquet-cpp.git
WORKDIR /parquet-cpp
-RUN ARROW_HOME=/usr THRIFT_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
+RUN ARROW_HOME=/usr /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON .
RUN make -j5 install
http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/Dockerfile-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index e62a601..29e00b0 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -16,32 +16,23 @@ FROM quay.io/pypa/manylinux1_x86_64:latest
RUN yum install -y flex openssl-devel
WORKDIR /
-ADD http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz /boost_1_60_0.tar.gz
+RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
RUN tar xf boost_1_60_0.tar.gz
WORKDIR /boost_1_60_0
RUN ./bootstrap.sh
RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system install
WORKDIR /
-ADD https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz /cmake-3.5.2.tar.gz
-RUN tar xf cmake-3.5.2.tar.gz
-WORKDIR /cmake-3.5.2
-RUN ./configure --prefix=/usr
-RUN make -j5 install
-
-WORKDIR /
-ADD https://github.com/google/flatbuffers/archive/v1.3.0.tar.gz /flatbuffers-1.3.0.tar.gz
-RUN tar xf flatbuffers-1.3.0.tar.gz
-WORKDIR /flatbuffers-1.3.0
-RUN CXXFLAGS='-fPIC' cmake -DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .
-RUN make -j5 install
+# Install cmake manylinux1 package
+RUN /opt/python/cp35-cp35m/bin/pip install cmake
WORKDIR /
RUN git clone https://github.com/matthew-brett/multibuild.git
WORKDIR /multibuild
RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
+
ADD arrow /arrow
WORKDIR /arrow/cpp
-RUN FLATBUFFERS_HOME=/usr cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF .
+RUN /opt/python/cp35-cp35m/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DARROW_HDFS=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON .
RUN make -j5 install
http://git-wip-us.apache.org/repos/asf/arrow/blob/b1472305/python/manylinux1/build_arrow.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 0786b6f..7e2ad58 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -30,10 +30,12 @@ source /multibuild/manylinux_utils.sh
cd /arrow/python
# PyArrow build configuration
-export PYARROW_CMAKE_OPTIONS='-DCMAKE_BUILD_TYPE=Release'
+export PYARROW_BUILD_TYPE='release'
+export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_PARQUET=ON'
# Need as otherwise arrow_io is sometimes not linked
export LDFLAGS="-Wl,--no-as-needed"
export ARROW_HOME="/usr"
+export PARQUET_HOME="/usr"
# Ensure the target directory exists
mkdir -p /io/dist
@@ -53,7 +55,7 @@ function repair_wheelhouse {
cp $whl $out_dir
else
# Store libraries directly in . not .libs to fix problems with libpyarrow.so linkage.
- auditwheel -v repair -L . $whl -w $out_dir/
+ $PY35_BIN/auditwheel -v repair -L . $whl -w $out_dir/
fi
done
chmod -R a+rwX $out_dir
@@ -67,10 +69,10 @@ for PYTHON in ${PYTHON_VERSIONS}; do
$PIPI_IO "numpy==1.9.0"
$PIPI_IO "cython==0.24"
+ $PIPI_IO "cmake"
PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
- rm_mkdir fixed_wheels
repair_wheelhouse dist /io/dist
done