You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/06 21:58:20 UTC
arrow git commit: ARROW-947: [Python] Improve execution time of
manylinux1 build
Repository: arrow
Updated Branches:
refs/heads/master bd36f6f59 -> 03c242c7c
ARROW-947: [Python] Improve execution time of manylinux1 build
Got it down to 14min: https://travis-ci.org/xhochy/arrow/builds/229461571
We can probably squeeze out 1-2 minutes more but that won't be easy. Build times are probably longer here as we build in release mode.
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #648 from xhochy/ARROW-947 and squashes the following commits:
ebe7507 [Uwe L. Korn] Use this branch as docker tag
3203b22 [Uwe L. Korn] Move manylinux commands to script
a6397da [Uwe L. Korn] Reset docker tag to latest
f2b5909 [Uwe L. Korn] Explicit set e
352215f [Uwe L. Korn] Pre-build virtualenvs
c1a1c89 [Uwe L. Korn] Pre-install python packages
0e83522 [Uwe L. Korn] Fix image tags
60a0c8c [Uwe L. Korn] Add ccache to the image
3426195 [Uwe L. Korn] Add brotli
dc0802b [Uwe L. Korn] Add more dependencies
9cadeb5 [Uwe L. Korn] Move gtest and flatbuffers to scripts
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/03c242c7
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/03c242c7
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/03c242c7
Branch: refs/heads/master
Commit: 03c242c7c614a4ca2089ea378796c38ae1d9fa3a
Parents: bd36f6f
Author: Uwe L. Korn <uw...@xhochy.com>
Authored: Sat May 6 17:58:10 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sat May 6 17:58:10 2017 -0400
----------------------------------------------------------------------
.travis.yml | 10 ++---
ci/travis_script_manylinux.sh | 21 +++++++++++
python/manylinux1/Dockerfile-x86_64 | 2 +-
python/manylinux1/Dockerfile-x86_64_base | 41 +++++++++++++--------
python/manylinux1/build_arrow.sh | 17 +++------
python/manylinux1/scripts/build_brotli.sh | 30 +++++++++++++++
python/manylinux1/scripts/build_ccache.sh | 21 +++++++++++
python/manylinux1/scripts/build_flatbuffers.sh | 21 +++++++++++
python/manylinux1/scripts/build_gtest.sh | 21 +++++++++++
python/manylinux1/scripts/build_snappy.sh | 22 +++++++++++
python/manylinux1/scripts/build_thrift.sh | 37 +++++++++++++++++++
python/manylinux1/scripts/build_virtualenvs.sh | 41 +++++++++++++++++++++
12 files changed, 248 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index d821b5a..e694162 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -56,13 +56,9 @@ matrix:
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
- language: cpp
before_script:
- - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
- script: |
- pushd python/manylinux1
- git clone ../../ arrow
- docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
- docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
- ls -l dist/
+ - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+ script:
+ - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
- language: java
os: linux
jdk: oraclejdk7
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/ci/travis_script_manylinux.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
new file mode 100755
index 0000000..69feb68
--- /dev/null
+++ b/ci/travis_script_manylinux.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+
+set -ex
+
+pushd python/manylinux1
+git clone ../../ arrow
+docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/Dockerfile-x86_64
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
index 8f55ba7..08fecb0 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -9,7 +9,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-947
ADD arrow /arrow
WORKDIR /arrow/cpp
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/Dockerfile-x86_64_base
----------------------------------------------------------------------
diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
index e38296d..2ae7e0f 100644
--- a/python/manylinux1/Dockerfile-x86_64_base
+++ b/python/manylinux1/Dockerfile-x86_64_base
@@ -28,25 +28,34 @@ WORKDIR /
RUN /opt/python/cp35-cp35m/bin/pip install cmake
RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
+ADD scripts/build_gtest.sh /
+RUN /build_gtest.sh
+ENV GTEST_HOME /googletest-release-1.7.0
+
+ADD scripts/build_flatbuffers.sh /
+RUN /build_flatbuffers.sh
+ENV FLATBUFFERS_HOME /usr
+
+ADD scripts/build_thrift.sh /
+RUN /build_thrift.sh
+ENV THRIFT_HOME /usr
+
+ADD scripts/build_brotli.sh /
+RUN /build_brotli.sh
+ENV BROTLI_HOME /usr
+
+ADD scripts/build_snappy.sh /
+RUN /build_snappy.sh
+ENV SNAPPY_HOME /usr
+
+ADD scripts/build_ccache.sh /
+RUN /build_ccache.sh
+
WORKDIR /
RUN git clone https://github.com/matthew-brett/multibuild.git
WORKDIR /multibuild
RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963
-
-WORKDIR /
-RUN wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
-RUN tar xf googletest-release-1.7.0.tar.gz
-WORKDIR /googletest-release-1.7.0
-RUN cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON .
-RUN make -j5
-ENV GTEST_HOME /googletest-release-1.7.0
-
WORKDIR /
-RUN wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
-RUN tar xf flatbuffers-1.6.0.tar.gz
-WORKDIR /flatbuffers-1.6.0
-RUN cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
-RUN make -j5
-RUN make install
-ENV FLATBUFFERS_HOME /usr
+ADD scripts/build_virtualenvs.sh /
+RUN /build_virtualenvs.sh
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/build_arrow.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index a11d3d4..e072749 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -27,6 +27,9 @@ MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
source /multibuild/manylinux_utils.sh
+# Quit on failure
+set -e
+
cd /arrow/python
# PyArrow build configuration
@@ -48,11 +51,6 @@ for PYTHON in ${PYTHON_VERSIONS}; do
PIPI_IO="$PIP install -f $MANYLINUX_URL"
PATH="$PATH:$(cpython_path $PYTHON)"
- echo "=== (${PYTHON}) Installing build dependencies ==="
- $PIPI_IO "numpy==1.9.0"
- $PIPI_IO "cython==0.25.2"
- $PIPI_IO "pandas==0.19.2"
-
echo "=== (${PYTHON}) Building Arrow C++ libraries ==="
ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}
mkdir -p "${ARROW_BUILD_DIR}"
@@ -77,14 +75,9 @@ for PYTHON in ${PYTHON_VERSIONS}; do
auditwheel -v repair -L . dist/pyarrow-*.whl -w repaired_wheels/
echo "=== (${PYTHON}) Testing manylinux1 wheel ==="
- # Fix version to keep build reproducible"
- $PIPI_IO "virtualenv==15.1.0"
- rm -rf venv
- "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download venv
- source ./venv/bin/activate
+ source /venv-test-${PYTHON}/bin/activate
pip install repaired_wheels/*.whl
- pip install pytest pandas
- py.test venv/lib/*/site-packages/pyarrow
+ py.test /venv-test-${PYTHON}/lib/*/site-packages/pyarrow
deactivate
mv repaired_wheels/*.whl /io/dist
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_brotli.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
new file mode 100755
index 0000000..4b4cbf1
--- /dev/null
+++ b/python/manylinux1/scripts/build_brotli.sh
@@ -0,0 +1,30 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+export BROTLI_VERSION="0.6.0"
+wget "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -O brotli-${BROTLI_VERSION}.tar.gz
+tar xf brotli-${BROTLI_VERSION}.tar.gz
+pushd brotli-${BROTLI_VERSION}
+mkdir build
+pushd build
+cmake -DCMAKE_BUILD_TYPE=release \
+ "-DCMAKE_CXX_FLAGS=-fPIC" \
+ "-DCMAKE_C_FLAGS=-fPIC" \
+ -DCMAKE_INSTALL_PREFIX=/usr \
+ -DBUILD_SHARED_LIBS=OFF \
+ ..
+make -j5
+make install
+popd
+popd
+rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION}
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_ccache.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_ccache.sh b/python/manylinux1/scripts/build_ccache.sh
new file mode 100755
index 0000000..6ad5d29
--- /dev/null
+++ b/python/manylinux1/scripts/build_ccache.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+wget https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -O ccache-3.3.4.tar.bz2
+tar xf ccache-3.3.4.tar.bz2
+pushd ccache-3.3.4
+./configure --prefix=/usr
+make -j5
+make install
+popd
+rm -rf ccache-3.3.4.tar.bz2 ccache-3.3.4
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_flatbuffers.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_flatbuffers.sh b/python/manylinux1/scripts/build_flatbuffers.sh
new file mode 100755
index 0000000..7703855
--- /dev/null
+++ b/python/manylinux1/scripts/build_flatbuffers.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz
+tar xf flatbuffers-1.6.0.tar.gz
+pushd flatbuffers-1.6.0
+cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF"
+make -j5
+make install
+popd
+rm -rf flatbuffers-1.6.0.tar.gz flatbuffers-1.6.0
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_gtest.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh
new file mode 100755
index 0000000..3427bed
--- /dev/null
+++ b/python/manylinux1/scripts/build_gtest.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz
+tar xf googletest-release-1.7.0.tar.gz
+ls -l
+pushd googletest-release-1.7.0
+cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON .
+make -j5
+popd
+rm -rf googletest-release-1.7.0.tar.gz
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_snappy.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_snappy.sh b/python/manylinux1/scripts/build_snappy.sh
new file mode 100755
index 0000000..973b4ff
--- /dev/null
+++ b/python/manylinux1/scripts/build_snappy.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+export SNAPPY_VERSION="1.1.3"
+wget "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -O snappy-${SNAPPY_VERSION}.tar.gz
+tar xf snappy-${SNAPPY_VERSION}.tar.gz
+pushd snappy-${SNAPPY_VERSION}
+./configure --with-pic "--prefix=/usr" CXXFLAGS='-DNDEBUG -O2'
+make -j5
+make install
+popd
+rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION}
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_thrift.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_thrift.sh b/python/manylinux1/scripts/build_thrift.sh
new file mode 100755
index 0000000..1db7458
--- /dev/null
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -ex
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+export THRIFT_VERSION=0.10.0
+wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
+tar xf thrift-${THRIFT_VERSION}.tar.gz
+pushd thrift-${THRIFT_VERSION}
+mkdir build-tmp
+pushd build-tmp
+cmake -DCMAKE_BUILD_TYPE=release \
+ "-DCMAKE_CXX_FLAGS=-fPIC" \
+ "-DCMAKE_C_FLAGS=-fPIC" \
+ "-DCMAKE_INSTALL_PREFIX=/usr" \
+ "-DCMAKE_INSTALL_RPATH=/usr/lib" \
+ "-DBUILD_SHARED_LIBS=OFF" \
+ "-DBUILD_TESTING=OFF" \
+ "-DWITH_QT4=OFF" \
+ "-DWITH_C_GLIB=OFF" \
+ "-DWITH_JAVA=OFF" \
+ "-DWITH_PYTHON=OFF" \
+ "-DWITH_CPP=ON" \
+ "-DWITH_STATIC_LIB=ON" ..
+make -j5
+make install
+popd
+popd
+rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
http://git-wip-us.apache.org/repos/asf/arrow/blob/03c242c7/python/manylinux1/scripts/build_virtualenvs.sh
----------------------------------------------------------------------
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
new file mode 100755
index 0000000..ee8a827
--- /dev/null
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -0,0 +1,41 @@
+#!/bin/bash -e
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7 3.4 3.5 3.6}"
+
+# Package index with only manylinux1 builds
+MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
+
+source /multibuild/manylinux_utils.sh
+
+for PYTHON in ${PYTHON_VERSIONS}; do
+ PYTHON_INTERPRETER="$(cpython_path $PYTHON)/bin/python"
+ PIP="$(cpython_path $PYTHON)/bin/pip"
+ PIPI_IO="$PIP install -f $MANYLINUX_URL"
+ PATH="$PATH:$(cpython_path $PYTHON)"
+
+ echo "=== (${PYTHON}) Installing build dependencies ==="
+ $PIPI_IO "numpy==1.9.0"
+ $PIPI_IO "cython==0.25.2"
+ $PIPI_IO "pandas==0.20.1"
+ $PIPI_IO "virtualenv==15.1.0"
+
+ echo "=== (${PYTHON}) Preparing virtualenv for tests ==="
+ "$(cpython_path $PYTHON)/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}
+ source /venv-test-${PYTHON}/bin/activate
+ pip install pytest 'numpy==1.12.1' 'pandas==0.20.1'
+ deactivate
+done