You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/20 23:22:44 UTC
[arrow] branch master updated: ARROW-5650: [Python] Update
manylinux dependency versions
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d0d7245 ARROW-5650: [Python] Update manylinux dependency versions
d0d7245 is described below
commit d0d7245ac199c0a2c0117e995aa7ca470e54ad6d
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Thu Jun 20 18:22:36 2019 -0500
ARROW-5650: [Python] Update manylinux dependency versions
Author: Antoine Pitrou <an...@python.org>
Closes #4634 from pitrou/ARROW-5650-update-manylinux-deps and squashes the following commits:
5c36c6c2b <Antoine Pitrou> ARROW-5650: Update manylinux dependency versions
---
python/manylinux1/scripts/build_bison.sh | 13 ++++++++-----
python/manylinux1/scripts/build_boost.sh | 4 ++--
python/manylinux1/scripts/build_brotli.sh | 2 +-
python/manylinux1/scripts/build_ccache.sh | 13 ++++++++-----
python/manylinux1/scripts/build_double_conversion.sh | 6 +++---
python/manylinux1/scripts/build_glog.sh | 2 +-
python/manylinux1/scripts/build_gtest.sh | 2 +-
python/manylinux1/scripts/build_llvm.sh | 2 +-
python/manylinux1/scripts/build_lz4.sh | 5 +++--
python/manylinux1/scripts/build_openssl.sh | 3 ++-
python/manylinux1/scripts/build_protobuf.sh | 3 ++-
python/manylinux1/scripts/build_re2.sh | 5 +++--
python/manylinux1/scripts/build_virtualenvs.sh | 6 ++++--
python/manylinux1/scripts/build_zlib.sh | 2 +-
python/manylinux1/scripts/build_zstd.sh | 11 +++++++++--
python/manylinux1/scripts/install_cmake.sh | 8 ++++----
16 files changed, 53 insertions(+), 34 deletions(-)
diff --git a/python/manylinux1/scripts/build_bison.sh b/python/manylinux1/scripts/build_bison.sh
index 29cc0be..e6f82fa 100755
--- a/python/manylinux1/scripts/build_bison.sh
+++ b/python/manylinux1/scripts/build_bison.sh
@@ -16,11 +16,14 @@
# specific language governing permissions and limitations
# under the License.
-wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz
-tar xf bison-3.0.4.tar.gz
-pushd bison-3.0.4
+BISON_VERSION=3.3.2
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+
+wget http://ftp.gnu.org/gnu/bison/bison-${BISON_VERSION}.tar.gz
+tar xf bison-${BISON_VERSION}.tar.gz
+pushd bison-${BISON_VERSION}
./configure --prefix=/usr
-make -j4
+make -j$NCORES
make install
popd
-rm -rf bison-3.0.4 bison-3.0.4.tar.gz
+rm -rf bison-${BISON_VERSION} bison-${BISON_VERSION}.tar.gz
diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh
index 666968d..ef3aef2 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-BOOST_VERSION=1.66.0
+BOOST_VERSION=1.68.0
BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
@@ -26,7 +26,7 @@ mkdir /arrow_boost
pushd /boost_${BOOST_VERSION_UNDERSCORE}
./bootstrap.sh
./b2 -j${NCORES} tools/bcp
-./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format variant multiprecision/cpp_int /arrow_boost
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build predef algorithm locale format variant multiprecision/cpp_int /arrow_boost
popd
pushd /arrow_boost
diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh
index f26bf59..41774ba 100755
--- a/python/manylinux1/scripts/build_brotli.sh
+++ b/python/manylinux1/scripts/build_brotli.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-export BROTLI_VERSION="0.6.0"
+export BROTLI_VERSION="1.0.7"
curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz
tar xf brotli-${BROTLI_VERSION}.tar.gz
pushd brotli-${BROTLI_VERSION}
diff --git a/python/manylinux1/scripts/build_ccache.sh b/python/manylinux1/scripts/build_ccache.sh
index c27f508..9dc0336 100755
--- a/python/manylinux1/scripts/build_ccache.sh
+++ b/python/manylinux1/scripts/build_ccache.sh
@@ -16,14 +16,17 @@
# specific language governing permissions and limitations
# under the License.
-curl -sL https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -o ccache-3.3.4.tar.bz2
-tar xf ccache-3.3.4.tar.bz2
-pushd ccache-3.3.4
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+CCACHE_VERSION=3.6
+
+curl -sLO https://www.samba.org/ftp/ccache/ccache-${CCACHE_VERSION}.tar.bz2
+tar xf ccache-${CCACHE_VERSION}.tar.bz2
+pushd ccache-${CCACHE_VERSION}
./configure --prefix=/usr
-make -j5
+make -j$NCORES
make install
popd
-rm -rf ccache-3.3.4.tar.bz2 ccache-3.3.4
+rm -rf ccache-${CCACHE_VERSION}.tar.bz2 ccache-${CCACHE_VERSION}
# Initialize the config directory, otherwise the build sometimes fails.
mkdir /root/.ccache
diff --git a/python/manylinux1/scripts/build_double_conversion.sh b/python/manylinux1/scripts/build_double_conversion.sh
index 0e73c70..27dd829 100755
--- a/python/manylinux1/scripts/build_double_conversion.sh
+++ b/python/manylinux1/scripts/build_double_conversion.sh
@@ -16,14 +16,14 @@
# specific language governing permissions and limitations
# under the License.
-DC_VERSION=3.1.1
+DC_VERSION=3.1.4
curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz
tar xf double-conversion-${DC_VERSION}.tar.gz
-pushd double-conversion-3.1.1
+pushd double-conversion-${DC_VERSION}
cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -GNinja -DCMAKE_POSITION_INDEPENDENT_CODE=ON
ninja install
popd
-rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-3.1.1
+rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-${DC_VERSION}
diff --git a/python/manylinux1/scripts/build_glog.sh b/python/manylinux1/scripts/build_glog.sh
index 5964e35..46a23f6 100755
--- a/python/manylinux1/scripts/build_glog.sh
+++ b/python/manylinux1/scripts/build_glog.sh
@@ -32,5 +32,5 @@ cmake -DCMAKE_BUILD_TYPE=Release \
-GNinja .
ninja install
popd
-rm -rf glog-${GLOG_VERSION}.tar.gz.tar.gz glog-${GLOG_VERSION}
+rm -rf glog-${GLOG_VERSION}.tar.gz glog-${GLOG_VERSION}
diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh
index 723b59b..6fd1a5b 100755
--- a/python/manylinux1/scripts/build_gtest.sh
+++ b/python/manylinux1/scripts/build_gtest.sh
@@ -36,4 +36,4 @@ ninja install
popd
popd
-rm -rf googletest-release-${GTEST_VERSION}.tar.gz
+rm -rf googletest-release-${GTEST_VERSION}.tar.gz googletest-release-${GTEST_VERSION}
diff --git a/python/manylinux1/scripts/build_llvm.sh b/python/manylinux1/scripts/build_llvm.sh
index 299d867..b975961 100755
--- a/python/manylinux1/scripts/build_llvm.sh
+++ b/python/manylinux1/scripts/build_llvm.sh
@@ -43,4 +43,4 @@ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
ninja install
popd
popd
-rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}
+rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src
diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh
index 75f791c..cffb029 100755
--- a/python/manylinux1/scripts/build_lz4.sh
+++ b/python/manylinux1/scripts/build_lz4.sh
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
export LZ4_VERSION="1.8.3"
export PREFIX="/usr"
export CFLAGS="${CFLAGS} -O3 -fPIC"
@@ -24,8 +25,8 @@ curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ
tar xf lz4-${LZ4_VERSION}.tar.gz
pushd lz4-${LZ4_VERSION}
-make -j5 PREFIX=${PREFIX}
-make install PREFIX=$PREFIX
+make -j$NCORES PREFIX=${PREFIX}
+make install PREFIX=${PREFIX}
popd
rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION}
# We don't want to link against shared libs
diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh
index b07ebac..c0ee2e5 100755
--- a/python/manylinux1/scripts/build_openssl.sh
+++ b/python/manylinux1/scripts/build_openssl.sh
@@ -16,7 +16,8 @@
# specific language governing permissions and limitations
# under the License.
-OPENSSL_VERSION="1.0.2q"
+# XXX OpenSSL 1.1.1 needs Perl 5.10 to compile.
+OPENSSL_VERSION="1.0.2s"
NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz
diff --git a/python/manylinux1/scripts/build_protobuf.sh b/python/manylinux1/scripts/build_protobuf.sh
index 27d0d9a..7261973 100755
--- a/python/manylinux1/scripts/build_protobuf.sh
+++ b/python/manylinux1/scripts/build_protobuf.sh
@@ -16,13 +16,14 @@
# specific language governing permissions and limitations
# under the License.
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
PROTOBUF_VERSION="3.7.1"
curl -sL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz" -o protobuf-${PROTOBUF_VERSION}.tar.gz
tar xf protobuf-${PROTOBUF_VERSION}.tar.gz
pushd protobuf-${PROTOBUF_VERSION}
./configure --disable-shared --prefix=/usr "CXXFLAGS=-O2 -fPIC"
-make -j10
+make -j$NCORES
make install
popd
rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION}
diff --git a/python/manylinux1/scripts/build_re2.sh b/python/manylinux1/scripts/build_re2.sh
index f600de9..67902c1 100755
--- a/python/manylinux1/scripts/build_re2.sh
+++ b/python/manylinux1/scripts/build_re2.sh
@@ -16,7 +16,8 @@
# specific language governing permissions and limitations
# under the License.
-export RE2_VERSION="2019-01-01"
+export RE2_VERSION="2019-04-01"
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz
tar xf re2-${RE2_VERSION}.tar.gz
@@ -26,7 +27,7 @@ export CXXFLAGS="-fPIC ${CXXFLAGS}"
export CFLAGS="-fPIC ${CFLAGS}"
# Build shared libraries
-make prefix=/usr -j8 install
+make prefix=/usr -j${NCORES} install
popd
rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/lib/libre2.so*
diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh
index a737e2f..beaf133 100755
--- a/python/manylinux1/scripts/build_virtualenvs.sh
+++ b/python/manylinux1/scripts/build_virtualenvs.sh
@@ -33,7 +33,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})"
echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
- $PIP install "numpy==1.14.5" "cython==0.29.3" "virtualenv==16.3.0"
+ $PIP install "numpy==1.14.5" "cython==0.29.8" "virtualenv==16.3.0"
# Pandas requires numpy and cython
$PIP install "pandas==0.24.0"
@@ -44,7 +44,6 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
fi
fi
-
echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ==="
"$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH}
source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
@@ -56,6 +55,7 @@ done
find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';'
find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';'
# Only Python 3.6+ packages are stripable as they are built inside of the image
find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';'
find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';'
@@ -68,3 +68,5 @@ rm -rf /root/.cache
# venv, i.e. 216MiB in total
rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests
rm -rf /venv-test-*/lib/*/site-packages/pandas/tests
+# Remove unused Python versions
+rm -rf /opt/_internal/cpython-3.4*
diff --git a/python/manylinux1/scripts/build_zlib.sh b/python/manylinux1/scripts/build_zlib.sh
index 182a42f..272b6c4 100755
--- a/python/manylinux1/scripts/build_zlib.sh
+++ b/python/manylinux1/scripts/build_zlib.sh
@@ -20,7 +20,7 @@ curl -sL https://zlib.net/zlib-1.2.11.tar.gz -o /zlib-1.2.11.tar.gz
tar xf zlib-1.2.11.tar.gz
pushd zlib-1.2.11
./configure
-make -j4
+make -j8
make install
popd
rm -rf zlib-1.2.11.tar.gz zlib-1.2.11
diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh
index 1add13f..696499a 100755
--- a/python/manylinux1/scripts/build_zstd.sh
+++ b/python/manylinux1/scripts/build_zstd.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-export ZSTD_VERSION="1.3.8"
+export ZSTD_VERSION="1.4.0"
curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz
tar xf zstd-${ZSTD_VERSION}.tar.gz
@@ -24,7 +24,14 @@ pushd zstd-${ZSTD_VERSION}
mkdir build_cmake
pushd build_cmake
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ../build/cmake -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=/usr \
+ -DZSTD_BUILD_PROGRAMS=off \
+ -DZSTD_BUILD_SHARED=off \
+ -DZSTD_BUILD_STATIC=on \
+ -DZSTD_MULTITHREAD_SUPPORT=off \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=1 \
+ ../build/cmake
ninja install
popd
diff --git a/python/manylinux1/scripts/install_cmake.sh b/python/manylinux1/scripts/install_cmake.sh
index cda2fce..95b8a66 100755
--- a/python/manylinux1/scripts/install_cmake.sh
+++ b/python/manylinux1/scripts/install_cmake.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-/opt/python/cp35-cp35m/bin/pip install cmake ninja
-ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake
-ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja
-strip /opt/_internal/cpython-3.*/lib/python3.5/site-packages/cmake/data/bin/*
+/opt/python/cp37-cp37m/bin/pip install cmake ninja
+ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake
+ln -s /opt/python/cp37-cp37m/bin/ninja /usr/bin/ninja
+strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*