You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/06/25 14:45:17 UTC
[arrow] branch master updated: ARROW-2461: [Python] Build
manylinux2010 wheels
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 0568544 ARROW-2461: [Python] Build manylinux2010 wheels
0568544 is described below
commit 0568544531437366d3b6ded87f6f9ced8d749d72
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Tue Jun 25 16:45:03 2019 +0200
ARROW-2461: [Python] Build manylinux2010 wheels
Author: Antoine Pitrou <an...@python.org>
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #4675 from pitrou/ARROW-2461-manylinux2010 and squashes the following commits:
2f218d50d <Krisztián Szűcs> remove multibuild's clean_code function call
1186b52cb <Krisztián Szűcs> use FETCH_HEAD in the CI templates
17a4c7fc7 <Antoine Pitrou> Nits
cb5f28530 <Antoine Pitrou> Attempt to factor out package scripts
bcd0761f4 <Antoine Pitrou> ARROW-2461: Build manylinux2010 wheels
---
dev/tasks/conda-recipes/appveyor.yml | 2 +-
dev/tasks/conda-recipes/travis.linux.yml | 2 +-
dev/tasks/conda-recipes/travis.osx.yml | 2 +-
dev/tasks/docker-tests/circle.linux.yml | 2 +-
dev/tasks/docker-tests/travis.linux.yml | 2 +-
dev/tasks/gandiva-jars/travis.linux.yml | 2 +-
dev/tasks/gandiva-jars/travis.osx.yml | 2 +-
dev/tasks/linux-packages/travis.linux.arm64.yml | 2 +-
dev/tasks/linux-packages/travis.linux.yml | 2 +-
dev/tasks/python-wheels/appveyor.yml | 2 +-
.../{linux-test.sh => manylinux-test.sh} | 3 +-
.../{travis.linux.yml => travis.manylinux.yml} | 15 +--
dev/tasks/python-wheels/travis.osx.yml | 5 +-
dev/tasks/tasks.yml | 104 ++++++++++++++++---
docker-compose.yml | 22 +++++
python/.gitignore | 4 +-
python/manylinux1/build_arrow.sh | 2 +-
python/manylinux2010/Dockerfile-x86_64_base | 105 ++++++++++++++++++++
python/manylinux2010/README.md | 110 +++++++++++++++++++++
.../{manylinux1 => manylinux2010}/build_arrow.sh | 36 +++----
python/manylinux2010/scripts/build_boost.sh | 43 ++++++++
.../manylinux2010/scripts/build_brotli.sh | 44 ++++-----
.../manylinux2010/scripts/build_bz2.sh | 34 ++-----
.../manylinux2010/scripts/build_cares.sh | 44 ++++-----
.../manylinux2010/scripts/build_clang.sh | 53 +++++-----
.../scripts/build_double_conversion.sh | 38 +++----
.../manylinux2010/scripts/build_flatbuffers.sh | 42 +++-----
.../manylinux2010/scripts/build_gflags.sh | 48 ++++-----
.../manylinux2010/scripts/build_glog.sh | 43 ++++----
.../manylinux2010/scripts/build_grpc.sh | 51 +++++-----
python/manylinux2010/scripts/build_llvm.sh | 48 +++++++++
.../manylinux2010/scripts/build_lz4.sh | 43 +++-----
.../manylinux2010/scripts/build_openssl.sh | 36 +++----
.../manylinux2010/scripts/build_protobuf.sh | 39 +++-----
.../manylinux2010/scripts/build_rapidjson.sh | 47 ++++-----
.../manylinux2010/scripts/build_re2.sh | 35 +++----
.../manylinux2010/scripts/build_snappy.sh | 41 +++-----
python/manylinux2010/scripts/build_thrift.sh | 46 +++++++++
.../manylinux2010/scripts/build_uriparser.sh | 47 ++++-----
python/manylinux2010/scripts/build_virtualenvs.sh | 72 ++++++++++++++
.../manylinux2010/scripts/build_zlib.sh | 36 ++-----
.../manylinux2010/scripts/build_zstd.sh | 49 +++++----
.../scripts/check_arrow_visibility.sh | 37 +++----
.../manylinux2010/scripts/install_cmake.sh | 32 +-----
44 files changed, 895 insertions(+), 579 deletions(-)
diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml
index df4bbd6..5d677c6 100644
--- a/dev/tasks/conda-recipes/appveyor.yml
+++ b/dev/tasks/conda-recipes/appveyor.yml
@@ -45,7 +45,7 @@ test_script:
# Clone arrow
- git clone --no-checkout {{ arrow.remote }} arrow || exit /B
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
- - git -C arrow checkout {{ arrow.head }} || exit /B
+ - git -C arrow checkout FETCH_HEAD || exit /B
- pushd arrow\dev\tasks\conda-recipes
diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml
index 703ebf0..9e9f94f 100644
--- a/dev/tasks/conda-recipes/travis.linux.yml
+++ b/dev/tasks/conda-recipes/travis.linux.yml
@@ -49,7 +49,7 @@ install:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- pushd arrow/dev/tasks/conda-recipes
diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml
index 83837ca..3093cac 100644
--- a/dev/tasks/conda-recipes/travis.osx.yml
+++ b/dev/tasks/conda-recipes/travis.osx.yml
@@ -48,7 +48,7 @@ install:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- pushd arrow/dev/tasks/conda-recipes
diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml
index 47f1817..7b664ee 100644
--- a/dev/tasks/docker-tests/circle.linux.yml
+++ b/dev/tasks/docker-tests/circle.linux.yml
@@ -25,7 +25,7 @@ jobs:
- run: docker-compose -v
- run: git clone --no-checkout {{ arrow.remote }} arrow
- run: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - run: git -C arrow checkout {{ arrow.head }}
+ - run: git -C arrow checkout FETCH_HEAD
- run: git -C arrow submodule update --init --recursive
- run: |
pushd arrow
diff --git a/dev/tasks/docker-tests/travis.linux.yml b/dev/tasks/docker-tests/travis.linux.yml
index d55d207..7d910d7 100644
--- a/dev/tasks/docker-tests/travis.linux.yml
+++ b/dev/tasks/docker-tests/travis.linux.yml
@@ -45,7 +45,7 @@ before_install:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- git -C arrow submodule update --init --recursive
script:
diff --git a/dev/tasks/gandiva-jars/travis.linux.yml b/dev/tasks/gandiva-jars/travis.linux.yml
index a7b800f..8b3bc21 100644
--- a/dev/tasks/gandiva-jars/travis.linux.yml
+++ b/dev/tasks/gandiva-jars/travis.linux.yml
@@ -43,7 +43,7 @@ before_install:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
diff --git a/dev/tasks/gandiva-jars/travis.osx.yml b/dev/tasks/gandiva-jars/travis.osx.yml
index efe8fec..ac25b04 100644
--- a/dev/tasks/gandiva-jars/travis.osx.yml
+++ b/dev/tasks/gandiva-jars/travis.osx.yml
@@ -35,7 +35,7 @@ env:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
- brew update
diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml
index 853b706..7bb5ad0 100644
--- a/dev/tasks/linux-packages/travis.linux.arm64.yml
+++ b/dev/tasks/linux-packages/travis.linux.arm64.yml
@@ -40,7 +40,7 @@ env:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
script:
- pushd arrow/dev/tasks/linux-packages
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
index 937d4c7..7546587 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.yml
@@ -34,7 +34,7 @@ env:
before_script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
script:
- pushd arrow/dev/tasks/linux-packages
diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
index 9576e64..6ad31dd 100644
--- a/dev/tasks/python-wheels/appveyor.yml
+++ b/dev/tasks/python-wheels/appveyor.yml
@@ -36,7 +36,7 @@ build_script:
- git clone --no-checkout {{ arrow.remote }} %ARROW_SRC% || exit /B
- git -C %ARROW_SRC% fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
- - git -C %ARROW_SRC% checkout {{ arrow.head }} || exit /B
+ - git -C %ARROW_SRC% checkout FETCH_HEAD || exit /B
- git -C %ARROW_SRC% submodule update --init || exit /B
# Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC)
diff --git a/dev/tasks/python-wheels/linux-test.sh b/dev/tasks/python-wheels/manylinux-test.sh
similarity index 94%
copy from dev/tasks/python-wheels/linux-test.sh
copy to dev/tasks/python-wheels/manylinux-test.sh
index 1fdabc7..f5056fb 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/dev/tasks/python-wheels/manylinux-test.sh
@@ -20,7 +20,7 @@
set -e
# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
+pip install -q /arrow/python/$WHEEL_TAG/dist/*.whl
# Runs tests on installed distribution from an empty directory
python --version
@@ -34,6 +34,7 @@ import pyarrow.parquet
import pyarrow.plasma
if sys.version_info.major > 2:
+ import pyarrow.flight
import pyarrow.gandiva
"
diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.manylinux.yml
similarity index 84%
rename from dev/tasks/python-wheels/travis.linux.yml
rename to dev/tasks/python-wheels/travis.manylinux.yml
index 31e633e..e3670e4 100644
--- a/dev/tasks/python-wheels/travis.linux.yml
+++ b/dev/tasks/python-wheels/travis.manylinux.yml
@@ -39,7 +39,7 @@ before_script:
script:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
- mkdir -p dist
# build wheel
@@ -51,22 +51,25 @@ script:
# don't attempt to build it, because the base image by pypa is updated
# regularly and would cause undeterministic builds
- - docker-compose pull python-manylinux1
+ - BUILD_IMAGE=python-{{ wheel_tag }}
+ - docker-compose pull $BUILD_IMAGE
- docker-compose run
-e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }}
-e PYTHON_VERSION="{{ python_version }}"
-e UNICODE_WIDTH="{{ unicode_width }}"
- python-manylinux1
+ $BUILD_IMAGE
- popd
# test on multiple distributions
{%- for image in test_docker_images %}
- - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow {{ image }}
- /arrow/dev/tasks/python-wheels/linux-test.sh
+ - docker run -it --shm-size 2G --volume $(pwd)/arrow:/arrow
+ --env WHEEL_TAG="{{ wheel_tag }}"
+ {{ image }}
+ /arrow/dev/tasks/python-wheels/manylinux-test.sh
{%- endfor %}
# prepare for deployment
- - sudo mv arrow/python/manylinux1/dist/* dist/
+ - sudo mv arrow/python/{{ wheel_tag }}/dist/* dist/
deploy:
provider: releases
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index 2db3208..ac58cd8 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -36,7 +36,7 @@ before_install:
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- - git -C arrow checkout {{ arrow.head }}
+ - git -C arrow checkout FETCH_HEAD
# ARROW-3976 Old versions of git can cause failures when Homebrew prints a
# donation solicitation. Attempt to update git
@@ -60,9 +60,6 @@ before_install:
install:
- mkdir -p dist
- # multibuild's default clean_code function
- - clean_code arrow {{ arrow.head }}
-
# the following functions are defined in osx-build.sh
- build_wheel arrow
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 5e74a5e..875216c 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -28,11 +28,16 @@ groups:
- conda-win-vs2015-py36
- conda-win-vs2015-py37
wheel:
- - wheel-linux-cp27m
- - wheel-linux-cp27mu
- - wheel-linux-cp35m
- - wheel-linux-cp36m
- - wheel-linux-cp37m
+ - wheel-manylinux1-cp27m
+ - wheel-manylinux1-cp27mu
+ - wheel-manylinux1-cp35m
+ - wheel-manylinux1-cp36m
+ - wheel-manylinux1-cp37m
+ - wheel-manylinux2010-cp27m
+ - wheel-manylinux2010-cp27mu
+ - wheel-manylinux2010-cp35m
+ - wheel-manylinux2010-cp36m
+ - wheel-manylinux2010-cp37m
- wheel-osx-cp27m
- wheel-osx-cp35m
- wheel-osx-cp36m
@@ -151,65 +156,134 @@ tasks:
############################## Wheel Linux ##################################
- wheel-linux-cp27m:
+ wheel-manylinux1-cp27m:
ci: travis
platform: linux
- template: python-wheels/travis.linux.yml
+ template: python-wheels/travis.manylinux.yml
params:
python_version: 2.7
unicode_width: 16
+ wheel_tag: manylinux1
test_docker_images: []
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
- wheel-linux-cp27mu:
+ wheel-manylinux1-cp27mu:
ci: travis
platform: linux
- template: python-wheels/travis.linux.yml
+ template: python-wheels/travis.manylinux.yml
params:
python_version: 2.7
unicode_width: 32
+ wheel_tag: manylinux1
test_docker_images:
- python:2.7-slim # debian ucs4
artifacts:
- pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
- wheel-linux-cp35m:
+ wheel-manylinux1-cp35m:
ci: travis
platform: linux
- template: python-wheels/travis.linux.yml
+ template: python-wheels/travis.manylinux.yml
params:
python_version: 3.5
unicode_width: 16
+ wheel_tag: manylinux1
test_docker_images:
- python:3.5-slim
artifacts:
- pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl
- wheel-linux-cp36m:
+ wheel-manylinux1-cp36m:
ci: travis
platform: linux
- template: python-wheels/travis.linux.yml
+ template: python-wheels/travis.manylinux.yml
params:
python_version: 3.6
unicode_width: 16
+ wheel_tag: manylinux1
test_docker_images:
- python:3.6-slim
artifacts:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl
- wheel-linux-cp37m:
+ wheel-manylinux1-cp37m:
ci: travis
platform: linux
- template: python-wheels/travis.linux.yml
+ template: python-wheels/travis.manylinux.yml
params:
python_version: 3.7
unicode_width: 16
+ wheel_tag: manylinux1
test_docker_images:
- python:3.7-slim
artifacts:
- pyarrow-{no_rc_version}-cp37-cp37m-manylinux1_x86_64.whl
+ wheel-manylinux2010-cp27m:
+ ci: travis
+ platform: linux
+ template: python-wheels/travis.manylinux.yml
+ params:
+ python_version: 2.7
+ unicode_width: 16
+ wheel_tag: manylinux2010
+ test_docker_images: []
+ artifacts:
+ - pyarrow-{no_rc_version}-cp27-cp27m-manylinux2010_x86_64.whl
+
+ wheel-manylinux2010-cp27mu:
+ ci: travis
+ platform: linux
+ template: python-wheels/travis.manylinux.yml
+ params:
+ python_version: 2.7
+ unicode_width: 32
+ wheel_tag: manylinux2010
+ test_docker_images:
+ - python:2.7-slim # debian ucs4
+ artifacts:
+ - pyarrow-{no_rc_version}-cp27-cp27mu-manylinux2010_x86_64.whl
+
+ wheel-manylinux2010-cp35m:
+ ci: travis
+ platform: linux
+ template: python-wheels/travis.manylinux.yml
+ params:
+ python_version: 3.5
+ unicode_width: 16
+ wheel_tag: manylinux2010
+ test_docker_images:
+ - python:3.5-slim
+ artifacts:
+ - pyarrow-{no_rc_version}-cp35-cp35m-manylinux2010_x86_64.whl
+
+ wheel-manylinux2010-cp36m:
+ ci: travis
+ platform: linux
+ template: python-wheels/travis.manylinux.yml
+ params:
+ python_version: 3.6
+ unicode_width: 16
+ wheel_tag: manylinux2010
+ test_docker_images:
+ - python:3.6-slim
+ artifacts:
+ - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl
+
+ wheel-manylinux2010-cp37m:
+ ci: travis
+ platform: linux
+ template: python-wheels/travis.manylinux.yml
+ params:
+ python_version: 3.7
+ unicode_width: 16
+ wheel_tag: manylinux2010
+ test_docker_images:
+ - python:3.7-slim
+ artifacts:
+ - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl
+
############################## Wheel OSX ####################################
wheel-osx-cp27m:
diff --git a/docker-compose.yml b/docker-compose.yml
index 6ff601d..adad951 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -439,6 +439,28 @@ services:
- ./python/manylinux1:/io:delegated
command: /io/build_arrow.sh
+ python-manylinux2010:
+ # Usage:
+ # either build:
+ # $ docker-compose build python-manylinux2010
+ # or pull:
+ # $ docker-compose pull python-manylinux2010
+ # an then run:
+ # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010
+ image: quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest
+ build:
+ context: python/manylinux2010
+ dockerfile: Dockerfile-x86_64_base
+ shm_size: 2G
+ environment:
+ PYARROW_PARALLEL: 3
+ PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+ UNICODE_WIDTH: ${UNICODE_WIDTH:-16}
+ volumes:
+ - .:/arrow:delegated
+ - ./python/manylinux2010:/io:delegated
+ command: /io/build_arrow.sh
+
######################### Integration Tests #################################
# impala:
diff --git a/python/.gitignore b/python/.gitignore
index 8f08f93..8fdd7c4 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -36,8 +36,10 @@ htmlcov
.asv
pyarrow/_table_api.h
-# manylinux1 temporary files
+# manylinux temporary files
manylinux1/arrow
+nm_arrow.log
+visible_symbols.log
# plasma store
pyarrow/plasma_store_server
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 6c8a2c1..867d6ff 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -57,12 +57,12 @@ PIP="${CPYTHON_PATH}/bin/pip"
PATH="${PATH}:${CPYTHON_PATH}"
if [ "${PYTHON_VERSION}" != "2.7" ]; then
- # Gandiva is not supported on Python 2.7
export PYARROW_WITH_FLIGHT=1
export PYARROW_WITH_GANDIVA=1
export BUILD_ARROW_FLIGHT=ON
export BUILD_ARROW_GANDIVA=ON
else
+ # Flight and Gandiva are not supported on Python 2.7
export PYARROW_WITH_FLIGHT=0
export PYARROW_WITH_GANDIVA=0
export BUILD_ARROW_FLIGHT=OFF
diff --git a/python/manylinux2010/Dockerfile-x86_64_base b/python/manylinux2010/Dockerfile-x86_64_base
new file mode 100644
index 0000000..cd6d445
--- /dev/null
+++ b/python/manylinux2010/Dockerfile-x86_64_base
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM quay.io/pypa/manylinux2010_x86_64:latest
+
+# Install build dependencies
+RUN yum install -y xz bison ccache flex wget
+RUN yum clean all
+
+# Install up-to-date CMake and Ninja versions
+# A fresh CMake is required to recognize recent Boost versions...
+ADD scripts/install_cmake.sh /
+RUN /install_cmake.sh
+
+ADD scripts/build_zlib.sh /
+RUN /build_zlib.sh
+
+WORKDIR /
+RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 1a7f31be677185f2dface2643284846e14130c3f
+
+ADD scripts/build_llvm.sh /
+RUN /build_llvm.sh
+
+# clang is only used to precompile Gandiva bitcode
+ADD scripts/build_clang.sh /
+RUN /build_clang.sh
+
+ADD scripts/build_openssl.sh /
+RUN /build_openssl.sh
+
+ADD scripts/build_boost.sh /
+RUN /build_boost.sh
+
+ADD scripts/build_thrift.sh /
+RUN /build_thrift.sh
+ENV THRIFT_HOME /usr/local
+
+ADD scripts/build_gflags.sh /
+RUN /build_gflags.sh
+
+ADD scripts/build_protobuf.sh /
+RUN /build_protobuf.sh
+ENV PROTOBUF_HOME /usr/local
+
+ADD scripts/build_cares.sh /
+RUN /build_cares.sh
+
+ADD scripts/build_grpc.sh /
+RUN /build_grpc.sh
+
+ADD scripts/build_flatbuffers.sh /
+RUN /build_flatbuffers.sh
+ENV FLATBUFFERS_HOME /usr/local
+
+ADD scripts/build_brotli.sh /
+RUN /build_brotli.sh
+ENV BROTLI_HOME /usr/local
+
+ADD scripts/build_snappy.sh /
+RUN /build_snappy.sh
+ENV SNAPPY_HOME /usr/local
+
+ADD scripts/build_lz4.sh /
+RUN /build_lz4.sh
+ENV LZ4_HOME /usr/local
+
+ADD scripts/build_zstd.sh /
+RUN /build_zstd.sh
+ENV ZSTD_HOME /usr/local
+
+ADD scripts/build_glog.sh /
+RUN /build_glog.sh
+ENV GLOG_HOME /usr/local
+
+ADD scripts/build_double_conversion.sh /
+RUN /build_double_conversion.sh
+
+ADD scripts/build_rapidjson.sh /
+RUN /build_rapidjson.sh
+
+ADD scripts/build_re2.sh /
+RUN /build_re2.sh
+
+ADD scripts/build_uriparser.sh /
+RUN /build_uriparser.sh
+
+ADD scripts/build_bz2.sh /
+RUN /build_bz2.sh
+
+ADD scripts/build_virtualenvs.sh /
+RUN /build_virtualenvs.sh
diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md
new file mode 100644
index 0000000..db0e414
--- /dev/null
+++ b/python/manylinux2010/README.md
@@ -0,0 +1,110 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+## Manylinux2010 wheels for Apache Arrow
+
+This folder provides base Docker images and an infrastructure to build
+`manylinux2010` compatible Python wheels that should be installable on all
+Linux distributions published in last four years.
+
+The process is split up in two parts:
+
+1. There are base Docker images that contain the build dependencies for
+ Arrow. Those images do not need to be rebuilt frequently, and are hosted
+ on the public quay.io service.
+
+2. Based on on these images, there is a bash script (`build_arrow.sh`) that
+ the PyArrow wheels for all supported Python versions, and place them
+ in the `dist` folder.
+
+### Building PyArrow
+
+You can build the PyArrow wheels by running the following command in this
+directory (this is for Python 2.7 with unicode width 16, similarly you can pass
+in `PYTHON_VERSION="3.5"`, `PYTHON_VERSION="3.6"` or `PYTHON_VERSION="3.7"` or
+use `PYTHON_VERSION="2.7"` with `UNICODE_WIDTH=32`):
+
+```bash
+# Build the python packages
+docker run --env PYTHON_VERSION="2.7" --env UNICODE_WIDTH=16 --shm-size=2g --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow quay.io/ursa-labs/arrow_manylinux2010_x86_64_base:latest /io/build_arrow.sh
+# Now the new packages are located in the dist/ folder
+ls -l dist/
+```
+
+### Re-building the build image
+
+In case you want to make changes to the base Docker image (for example because
+you want to update a dependency to a new version), you must re-build it.
+The Docker configuration is in `Dockerfile-x86_64_base`, and it calls into
+scripts stored under the `scripts` directory.
+
+```bash
+docker build -t arrow_manylinux2010_x86_64_base -f Dockerfile-x86_64_base .
+```
+
+For each dependency, a bash script in the `scripts/` directory downloads the
+source code, builds it and installs the build results. At the end of each
+dependency build the sources are removed again so that only the binary
+installation of a dependency is persisted in the Docker image.
+
+### Publishing a new build image
+
+If you have write access to the `quay.io` Ursa Labs account, you can directly
+publish a build image that you built locally.
+
+For that you need to first tag your image for quay.io upload:
+```bash
+$ docker image tag arrow_manylinux2010_x86_64_base:latest quay.io/ursa-labs/arrow_manylinux2010_x86_64_base
+```
+
+Then you can push it:
+```bash
+$ docker image push quay.io/ursa-labs/arrow_manylinux2010_x86_64_base
+The push refers to repository [quay.io/ursa-labs/arrow_manylinux2010_x86_64_base]
+a1ab88d27acc: Pushing [==============> ] 492.5MB/1.645GB
+[... etc. ...]
+```
+
+### Using quay.io to trigger and build the docker image
+
+You can also create your own `quay.io` repository and trigger builds there from
+your Github fork of the Arrow repository.
+
+1. Make the change in the build scripts (eg. to modify the boost build, update `scripts/boost.sh`).
+
+2. Setup an account on quay.io and link to your GitHub account
+
+3. In quay.io, Add a new repository using :
+
+ 1. Link to GitHub repository push
+ 2. Trigger build on changes to a specific branch (eg. myquay) of the repo (eg. `pravindra/arrow`)
+ 3. Set Dockerfile location to `/python/manylinux2010/Dockerfile-x86_64_base`
+ 4. Set Context location to `/python/manylinux2010`
+
+4. Push change (in step 1) to the branch specified in step 3.ii
+
+ * This should trigger a build in quay.io, the build takes about 2 hrs to finish.
+
+5. Add a tag `latest` to the build after step 4 finishes, save the build ID (eg. `quay.io/pravindra/arrow_manylinux2010_x86_64_base:latest`)
+
+6. In your arrow PR,
+
+ * include the change from 1.
+ * modify the `python-manylinux2010` entry in `docker-compose.yml`
+ to switch to the location from step 5 for the docker image.
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux2010/build_arrow.sh
similarity index 81%
copy from python/manylinux1/build_arrow.sh
copy to python/manylinux2010/build_arrow.sh
index 6c8a2c1..d4268e5 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux2010/build_arrow.sh
@@ -44,7 +44,7 @@ export PYARROW_BUNDLE_BOOST=1
export PYARROW_BOOST_NAMESPACE=arrow_boost
export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig
-export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist'
+export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist'
# Ensure the target directory exists
mkdir -p /io/dist
@@ -57,12 +57,12 @@ PIP="${CPYTHON_PATH}/bin/pip"
PATH="${PATH}:${CPYTHON_PATH}"
if [ "${PYTHON_VERSION}" != "2.7" ]; then
- # Gandiva is not supported on Python 2.7
export PYARROW_WITH_FLIGHT=1
export PYARROW_WITH_GANDIVA=1
export BUILD_ARROW_FLIGHT=ON
export BUILD_ARROW_GANDIVA=ON
else
+ # Flight and Gandiva are not supported on Python 2.7
export PYARROW_WITH_FLIGHT=0
export PYARROW_WITH_GANDIVA=0
export BUILD_ARROW_FLIGHT=OFF
@@ -83,8 +83,9 @@ PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_LIBDIR=lib \
-DARROW_BUILD_TESTS=OFF \
-DARROW_BUILD_SHARED=ON \
+ -DARROW_BUILD_STATIC=OFF \
-DARROW_BOOST_USE_SHARED=ON \
- -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \
+ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/;-isystem;/opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/" \
-DARROW_JEMALLOC=ON \
-DARROW_RPATH_ORIGIN=ON \
-DARROW_PYTHON=ON \
@@ -123,19 +124,15 @@ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
# Source distribution is used for debian pyarrow packages.
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py sdist
-if [ -n "$UBUNTU_WHEELS" ]; then
- echo "=== (${PYTHON_VERSION}) Wheels are not compatible with manylinux1 ==="
- mv dist/pyarrow-*.whl /io/dist
-else
- echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux1 ==="
- mkdir -p repaired_wheels/
- auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels/
+echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux2010 ==="
+mkdir -p repaired_wheels/
+auditwheel repair --plat manylinux2010_x86_64 -L . dist/pyarrow-*.whl -w repaired_wheels/
- # Install the built wheels
- $PIP install repaired_wheels/*.whl
+# Install the built wheels
+$PIP install repaired_wheels/*.whl
- # Test that the modules are importable
- $PYTHON_INTERPRETER -c "
+# Test that the modules are importable
+$PYTHON_INTERPRETER -c "
import sys
import pyarrow
import pyarrow.orc
@@ -145,10 +142,9 @@ import pyarrow.plasma
if sys.version_info.major > 2:
import pyarrow.flight
import pyarrow.gandiva
- "
+"
- # More thorough testing happens outsite of the build to prevent
- # packaging issues like ARROW-4372
- mv dist/*.tar.gz /io/dist
- mv repaired_wheels/*.whl /io/dist
-fi
+# More thorough testing happens outsite of the build to prevent
+# packaging issues like ARROW-4372
+mv dist/*.tar.gz /io/dist
+mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux2010/scripts/build_boost.sh b/python/manylinux2010/scripts/build_boost.sh
new file mode 100755
index 0000000..ef3aef2
--- /dev/null
+++ b/python/manylinux2010/scripts/build_boost.sh
@@ -0,0 +1,43 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+BOOST_VERSION=1.68.0
+BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+
+curl -sL https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz -o /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+mkdir /arrow_boost
+pushd /boost_${BOOST_VERSION_UNDERSCORE}
+./bootstrap.sh
+./b2 -j${NCORES} tools/bcp
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build predef algorithm locale format variant multiprecision/cpp_int /arrow_boost
+popd
+
+pushd /arrow_boost
+ls -l
+./bootstrap.sh
+./bjam -j${NCORES} dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install
+popd
+rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
+# Boost always install header-only parts but they also take up quite some space.
+# We don't need them in array, so don't persist them in the docker layer.
+# fusion 16.7 MiB
+rm -r /arrow_boost_dist/include/boost/fusion
+# spirit 8.2 MiB
+rm -r /arrow_boost_dist/include/boost/spirit
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_brotli.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_brotli.sh
index 1fdabc7..58715ca 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_brotli.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,19 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export BROTLI_VERSION="1.0.7"
+curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz
+tar xf brotli-${BROTLI_VERSION}.tar.gz
+pushd brotli-${BROTLI_VERSION}
+mkdir build
+pushd build
+cmake -DCMAKE_BUILD_TYPE=release \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DBUILD_SHARED_LIBS=OFF \
+ -GNinja \
+ ..
+ninja install
+popd
+popd
+rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_bz2.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_bz2.sh
index 1fdabc7..75ff7a2 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_bz2.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,15 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+export BZ2_VERSION="1.0.6"
+export CFLAGS="-Wall -Winline -O2 -fPIC -D_FILE_OFFSET_BITS=64"
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
+curl -sL "https://www.sourceware.org/pub/bzip2/bzip2-${BZ2_VERSION}.tar.gz" -o bzip2-${BZ2_VERSION}.tar.gz
+tar xf bzip2-${BZ2_VERSION}.tar.gz
-export ARROW_TEST_DATA=/arrow/testing/data
+pushd bzip2-${BZ2_VERSION}
+make PREFIX=/usr/local CFLAGS="$CFLAGS" install -j${NCORES}
+popd
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+rm -rf bzip2-${BZ2_VERSION}.tar.gz bzip2-${BZ2_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_cares.sh
similarity index 59%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_cares.sh
index 1fdabc7..a3897d7 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_cares.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,19 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export CARES_VERSION="1.15.0"
+export CFLAGS="-fPIC"
+export PREFIX="/usr/local"
+curl -sL "https://c-ares.haxx.se/download/c-ares-$CARES_VERSION.tar.gz" -o c-ares-${CARES_VERSION}.tar.gz
+tar xf c-ares-${CARES_VERSION}.tar.gz
+pushd c-ares-${CARES_VERSION}
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=${PREFIX} \
+ -DCARES_STATIC=ON \
+ -DCARES_SHARED=OFF \
+ -DCMAKE_C_FLAGS=${CFLAGS} \
+ -GNinja .
+ninja install
+popd
+rm -rf c-ares-${CARES_VERSION}.tar.gz c-ares-${CARES_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_clang.sh
similarity index 50%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_clang.sh
index 1fdabc7..9708029 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_clang.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,28 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+source /multibuild/manylinux_utils.sh
+
+export LLVM_VERSION="7.0.1"
+curl -sL http://releases.llvm.org/${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz -o cfe-${LLVM_VERSION}.src.tar.xz
+unxz cfe-${LLVM_VERSION}.src.tar.xz
+tar xf cfe-${LLVM_VERSION}.src.tar
+pushd cfe-${LLVM_VERSION}.src
+mkdir build
+pushd build
+cmake \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCLANG_INCLUDE_TESTS=OFF \
+ -DCLANG_INCLUDE_DOCS=OFF \
+ -DCLANG_ENABLE_ARCMT=OFF \
+ -DCLANG_ENABLE_STATIC_ANALYZER=OFF \
+ -DLLVM_ENABLE_RTTI=ON \
+ -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \
+ -DLLVM_INCLUDE_TESTS=OFF \
+ -DLLVM_INCLUDE_DOCS=OFF \
+ -GNinja \
+ ..
+ninja install
+popd
+popd
+rm -rf cfe-${LLVM_VERSION}.src.tar.xz cfe-${LLVM_VERSION}.src.tar cfe-${LLVM_VERSION}.src
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_double_conversion.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_double_conversion.sh
index 1fdabc7..bf4ea07 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_double_conversion.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,19 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
+DC_VERSION=3.1.4
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
+curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz
+tar xf double-conversion-${DC_VERSION}.tar.gz
+pushd double-conversion-${DC_VERSION}
-export ARROW_TEST_DATA=/arrow/testing/data
+cmake . \
+ -GNinja \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF
+ninja install
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+popd
+rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-${DC_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_flatbuffers.sh
similarity index 59%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_flatbuffers.sh
index 1fdabc7..f5af7cc 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_flatbuffers.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,17 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export FLATBUFFERS_VERSION=1.10.0
+curl -sL https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz \
+ -o flatbuffers-${FLATBUFFERS_VERSION}.tar.gz
+tar xf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz
+pushd flatbuffers-${FLATBUFFERS_VERSION}
+cmake \
+ -DCMAKE_CXX_FLAGS=-fPIC \
+ -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \
+ -DFLATBUFFERS_BUILD_TESTS=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -GNinja .
+ninja install
+popd
+rm -rf flatbuffers-${FLATBUFFERS_VERSION}.tar.gz flatbuffers-${FLATBUFFERS_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_gflags.sh
similarity index 58%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_gflags.sh
index 1fdabc7..7f40437 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_gflags.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,23 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export GFLAGS_VERSION="2.2.1"
+export CFLAGS="-fPIC"
+export CXXFLAGS="-fPIC"
+
+curl -sL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz" -o gflags-${GFLAGS_VERSION}.tar.gz
+tar xf gflags-${GFLAGS_VERSION}.tar.gz
+pushd gflags-${GFLAGS_VERSION}
+
+cmake . \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=/usr \
+ -DINSTALL_HEADERS=on \
+ -DBUILD_SHARED_LIBS=off \
+ -DBUILD_STATIC_LIBS=on \
+ -DBUILD_TESTING=off \
+ -GNinja
+
+ninja install
+popd
+rm -rf gflags-${GFLAGS_VERSION}.tar.gz gflags-${GFLAGS_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_glog.sh
similarity index 59%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_glog.sh
index 1fdabc7..5e3b8c2 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_glog.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,20 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
+export GLOG_VERSION="0.3.5"
+export PREFIX="/usr/local"
+curl -sL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz" -o glog-${GLOG_VERSION}.tar.gz
+tar xf glog-${GLOG_VERSION}.tar.gz
+pushd glog-${GLOG_VERSION}
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=${PREFIX} \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=1 \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTING=OFF \
+ -DWITH_GFLAGS=OFF \
+ -GNinja .
+ninja install
+popd
+rm -rf glog-${GLOG_VERSION}.tar.gz glog-${GLOG_VERSION}
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_grpc.sh
similarity index 50%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_grpc.sh
index 1fdabc7..6b90686 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_grpc.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,26 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export GRPC_VERSION="1.20.0"
+export CFLAGS="-fPIC -DGPR_MANYLINUX1=1"
+export PREFIX="/usr/local"
+
+curl -sL "https://github.com/grpc/grpc/archive/v${GRPC_VERSION}.tar.gz" -o grpc-${GRPC_VERSION}.tar.gz
+tar xf grpc-${GRPC_VERSION}.tar.gz
+pushd grpc-${GRPC_VERSION}
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=${PREFIX} \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DCMAKE_C_FLAGS="${CFLAGS}" \
+ -DCMAKE_CXX_FLAGS="${CFLAGS}" \
+ -DgRPC_CARES_PROVIDER=package \
+ -DgRPC_GFLAGS_PROVIDER=package \
+ -DgRPC_PROTOBUF_PROVIDER=package \
+ -DgRPC_SSL_PROVIDER=package \
+ -DgRPC_ZLIB_PROVIDER=package \
+ -DOPENSSL_USE_STATIC_LIBS=ON \
+ -GNinja .
+ninja install
+popd
+rm -rf grpc-${GRPC_VERSION}.tar.gz grpc-${GRPC_VERSION}
diff --git a/python/manylinux2010/scripts/build_llvm.sh b/python/manylinux2010/scripts/build_llvm.sh
new file mode 100755
index 0000000..1cc6645
--- /dev/null
+++ b/python/manylinux2010/scripts/build_llvm.sh
@@ -0,0 +1,48 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source /multibuild/manylinux_utils.sh
+
+LLVM_VERSION="7.0.1"
+PREFIX="/usr/local"
+
+curl -sL http://releases.llvm.org/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz
+unxz llvm-${LLVM_VERSION}.src.tar.xz
+tar xf llvm-${LLVM_VERSION}.src.tar
+pushd llvm-${LLVM_VERSION}.src
+mkdir build
+pushd build
+cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_TARGETS_TO_BUILD=host \
+ -DLLVM_INCLUDE_DOCS=OFF \
+ -DLLVM_INCLUDE_EXAMPLES=OFF \
+ -DLLVM_INCLUDE_TESTS=OFF \
+ -DLLVM_INCLUDE_UTILS=OFF \
+ -DLLVM_ENABLE_TERMINFO=OFF \
+ -DLLVM_ENABLE_ASSERTIONS=ON \
+ -DLLVM_ENABLE_RTTI=ON \
+ -DLLVM_ENABLE_OCAMLDOC=OFF \
+ -DLLVM_USE_INTEL_JITEVENTS=ON \
+ -DPYTHON_EXECUTABLE="$(cpython_path 2.7 32)/bin/python" \
+ -GNinja \
+ ..
+ninja install
+popd
+popd
+rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_lz4.sh
similarity index 57%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_lz4.sh
index 1fdabc7..7b3d441 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_lz4.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,18 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+export LZ4_VERSION="1.8.3"
+export PREFIX="/usr/local"
+export CFLAGS="${CFLAGS} -O3 -fPIC"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib -L${PREFIX}/lib"
+curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ4_VERSION}.tar.gz
+tar xf lz4-${LZ4_VERSION}.tar.gz
+pushd lz4-${LZ4_VERSION}
+
+make -j$NCORES PREFIX=${PREFIX}
+make install PREFIX=${PREFIX}
+popd
+rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION}
+# We don't want to link against shared libs
+rm -rf /usr/lib/liblz4.so*
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_openssl.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_openssl.sh
index 1fdabc7..d9ce0a8 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_openssl.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,17 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
+# XXX OpenSSL 1.1.1 needs Perl 5.10 to compile.
+OPENSSL_VERSION="1.1.1c"
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
+wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz
+tar xf openssl-${OPENSSL_VERSION}.tar.gz
-export ARROW_TEST_DATA=/arrow/testing/data
+pushd openssl-${OPENSSL_VERSION}
+./config -fpic no-shared no-tests --prefix=/usr/local
+make -j${NCORES}
+make install_sw
+popd
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+rm -rf openssl-${OPENSSL_VERSION}.tar.gz openssl-${OPENSSL_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_protobuf.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_protobuf.sh
index 1fdabc7..abb31c2 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_protobuf.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,14 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
+PROTOBUF_VERSION="3.7.1"
+
+curl -sL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz" -o protobuf-${PROTOBUF_VERSION}.tar.gz
+tar xf protobuf-${PROTOBUF_VERSION}.tar.gz
+pushd protobuf-${PROTOBUF_VERSION}
+./configure --disable-shared --prefix=/usr/local "CXXFLAGS=-O2 -fPIC"
+make -j$NCORES
+make install
+popd
+rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_rapidjson.sh
similarity index 55%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_rapidjson.sh
index 1fdabc7..4a93bd4 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_rapidjson.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,22 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export RAPIDJSON_VERSION="1.1.0"
+
+curl -sL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz" -o rapidjson-${RAPIDJSON_VERSION}.tar.gz
+tar xf rapidjson-${RAPIDJSON_VERSION}.tar.gz
+pushd rapidjson-${RAPIDJSON_VERSION}
+mkdir build
+pushd build
+cmake -GNinja \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DRAPIDJSON_HAS_STDSTRING=ON \
+ -DRAPIDJSON_BUILD_TESTS=OFF \
+ -DRAPIDJSON_BUILD_EXAMPLES=OFF \
+ -DRAPIDJSON_BUILD_DOC=OFF \
+ -DCMAKE_BUILD_TYPE=release \
+ ..
+ninja install
+popd
+popd
+rm -rf rapidjson-${RAPIDJSON_VERSION}.tar.gz rapidjson-${RAPIDJSON_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_re2.sh
similarity index 58%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_re2.sh
index 1fdabc7..143d8dc 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_re2.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,20 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
+export RE2_VERSION="2019-04-01"
+NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1))
-# Runs tests on installed distribution from an empty directory
-python --version
+curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz
+tar xf re2-${RE2_VERSION}.tar.gz
+pushd re2-${RE2_VERSION}
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
+export CXXFLAGS="-fPIC -O2 ${CXXFLAGS}"
+export CFLAGS="-fPIC -O2 ${CFLAGS}"
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
+# Build shared libraries
+make prefix=/usr/local -j${NCORES} install
-export ARROW_TEST_DATA=/arrow/testing/data
+popd
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+# Need to remove shared library to make sure the static library is picked up by Arrow
+rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/local/lib/libre2.so*
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_snappy.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_snappy.sh
index 1fdabc7..a67ac7f 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_snappy.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,16 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export SNAPPY_VERSION="1.1.7"
+curl -sL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz
+tar xf snappy-${SNAPPY_VERSION}.tar.gz
+pushd snappy-${SNAPPY_VERSION}
+CXXFLAGS='-DNDEBUG -O2' cmake -GNinja \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=1 \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DSNAPPY_BUILD_TESTS=OFF \
+ .
+ninja install
+popd
+rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION}
diff --git a/python/manylinux2010/scripts/build_thrift.sh b/python/manylinux2010/scripts/build_thrift.sh
new file mode 100755
index 0000000..3a0b9e8
--- /dev/null
+++ b/python/manylinux2010/scripts/build_thrift.sh
@@ -0,0 +1,46 @@
+#!/bin/bash -ex
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export THRIFT_VERSION=0.12.0
+wget http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz
+tar xf thrift-${THRIFT_VERSION}.tar.gz
+pushd thrift-${THRIFT_VERSION}
+mkdir build-tmp
+pushd build-tmp
+cmake -DCMAKE_BUILD_TYPE=release \
+ -DCMAKE_CXX_FLAGS=-fPIC \
+ -DCMAKE_C_FLAGS=-fPIC \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DCMAKE_INSTALL_RPATH=/usr/local/lib \
+ -DBUILD_EXAMPLES=OFF \
+ -DBUILD_TESTING=OFF \
+ -DWITH_QT4=OFF \
+ -DWITH_C_GLIB=OFF \
+ -DWITH_JAVA=OFF \
+ -DWITH_PYTHON=OFF \
+ -DWITH_HASKELL=OFF \
+ -DWITH_CPP=ON \
+ -DWITH_STATIC_LIB=ON \
+ -DWITH_SHARED_LIB=OFF \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT=/arrow_boost_dist \
+ -GNinja ..
+ninja install
+popd
+popd
+rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_uriparser.sh
similarity index 51%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_uriparser.sh
index 1fdabc7..840df5a 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_uriparser.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,22 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export URIPARSER_VERSION="0.9.2"
+export CFLAGS="-fPIC"
+export PREFIX="/usr/local"
+curl -sL "https://github.com/uriparser/uriparser/archive/uriparser-${URIPARSER_VERSION}.tar.gz" -o uriparser-${URIPARSER_VERSION}.tar.gz
+tar xf uriparser-${URIPARSER_VERSION}.tar.gz
+pushd uriparser-uriparser-${URIPARSER_VERSION}
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=${PREFIX} \
+ -DURIPARSER_BUILD_DOCS=off \
+ -DURIPARSER_BUILD_TESTS=off \
+ -DURIPARSER_BUILD_TOOLS=off \
+ -DURIPARSER_BUILD_WCHAR_T=off \
+ -DBUILD_SHARED_LIBS=off \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=on \
+ -GNinja .
+ninja install
+popd
+rm -rf uriparser-${URIPARSER_VERSION}.tar.gz uriparser-uriparser-${URIPARSER_VERSION}
diff --git a/python/manylinux2010/scripts/build_virtualenvs.sh b/python/manylinux2010/scripts/build_virtualenvs.sh
new file mode 100755
index 0000000..67ec961
--- /dev/null
+++ b/python/manylinux2010/scripts/build_virtualenvs.sh
@@ -0,0 +1,72 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build upon the scripts in https://github.com/matthew-brett/manylinux-builds
+# * Copyright (c) 2013-2016, Matt Terry and Matthew Brett (BSD 2-clause)
+
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-2.7,16 2.7,32 3.5,16 3.6,16, 3.7,16}"
+
+source /multibuild/manylinux_utils.sh
+
+for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
+ IFS=","
+ set -- $PYTHON_TUPLE;
+ PYTHON=$1
+ U_WIDTH=$2
+ PYTHON_INTERPRETER="$(cpython_path $PYTHON ${U_WIDTH})/bin/python"
+ PIP="$(cpython_path $PYTHON ${U_WIDTH})/bin/pip"
+ PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})"
+
+ echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ==="
+ $PIP install "numpy==1.14.*" "cython==0.29.8" "virtualenv==16.3.0"
+ # Pandas requires numpy and cython
+ $PIP install "pandas==0.24.0"
+
+ # TensorFlow is not supported for Python 2.7 with unicode width 16 or with Python 3.7
+ if [ $PYTHON != "2.7" ] || [ $U_WIDTH = "32" ]; then
+ if [ $PYTHON != "3.7" ]; then
+ $PIP install "tensorflow==1.11.0" "Keras-Preprocessing==1.0.5"
+ fi
+ fi
+
+ echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ==="
+ "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH}
+ source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
+ pip install pytest hypothesis "numpy==1.14.*" "pandas==0.24.0"
+ deactivate
+done
+
+# Remove debug symbols from libraries that were installed via wheel.
+find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
+find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';'
+# Only Python 3.6+ packages are stripable as they are built inside of the image
+find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';'
+find /opt/_internal/*/lib/*/site-packages/Cython -name '*.so' -exec strip '{}' ';'
+
+# Remove pip cache again. It's useful during the virtualenv creation but we
+# don't want it persisted in the docker layer, ~264MiB
+rm -rf /root/.cache
+# Remove pandas' tests module as it includes a lot of data, ~27MiB per Python
+# venv, i.e. 216MiB in total
+rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests
+rm -rf /venv-test-*/lib/*/site-packages/pandas/tests
+# Remove unused Python versions
+rm -rf /opt/_internal/cpython-3.4*
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_zlib.sh
similarity index 60%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_zlib.sh
index 1fdabc7..272b6c4 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_zlib.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,11 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+curl -sL https://zlib.net/zlib-1.2.11.tar.gz -o /zlib-1.2.11.tar.gz
+tar xf zlib-1.2.11.tar.gz
+pushd zlib-1.2.11
+./configure
+make -j8
+make install
+popd
+rm -rf zlib-1.2.11.tar.gz zlib-1.2.11
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/build_zstd.sh
similarity index 56%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/build_zstd.sh
index 1fdabc7..d6b0b09 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/build_zstd.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,24 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+export ZSTD_VERSION="1.4.0"
+
+curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz
+tar xf zstd-${ZSTD_VERSION}.tar.gz
+pushd zstd-${ZSTD_VERSION}
+mkdir build_cmake
+pushd build_cmake
+
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DZSTD_BUILD_PROGRAMS=off \
+ -DZSTD_BUILD_SHARED=off \
+ -DZSTD_BUILD_STATIC=on \
+ -DZSTD_MULTITHREAD_SUPPORT=off \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=1 \
+ ../build/cmake
+ninja install
+
+popd
+popd
+rm -rf zstd-${ZSTD_VERSION}.tar.gz zstd-${ZSTD_VERSION}
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/check_arrow_visibility.sh
similarity index 56%
copy from dev/tasks/python-wheels/linux-test.sh
copy to python/manylinux2010/scripts/check_arrow_visibility.sh
index 1fdabc7..ed464e0 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/check_arrow_visibility.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,20 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
+nm --demangle --dynamic /arrow-dist/lib/libarrow.so > nm_arrow.log
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
+# Filter out Arrow symbols and see if anything remains.
+# '_init' and '_fini' symbols may or not be present, we don't care.
+# (note we must ignore the grep exit status when no match is found)
+grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
+if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]]
+then
+ exit 0
+fi
-export ARROW_TEST_DATA=/arrow/testing/data
+echo "== Unexpected symbols exported by libarrow.so =="
+cat visible_symbols.log
+echo "================================================"
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+exit 1
diff --git a/dev/tasks/python-wheels/linux-test.sh b/python/manylinux2010/scripts/install_cmake.sh
similarity index 60%
rename from dev/tasks/python-wheels/linux-test.sh
rename to python/manylinux2010/scripts/install_cmake.sh
index 1fdabc7..637ea05 100755
--- a/dev/tasks/python-wheels/linux-test.sh
+++ b/python/manylinux2010/scripts/install_cmake.sh
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+#!/bin/bash -e
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,28 +16,7 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
-# Install built wheel
-pip install -q /arrow/python/manylinux1/dist/*.whl
-
-# Runs tests on installed distribution from an empty directory
-python --version
-
-# Test optional dependencies
-python -c "
-import sys
-import pyarrow
-import pyarrow.orc
-import pyarrow.parquet
-import pyarrow.plasma
-
-if sys.version_info.major > 2:
- import pyarrow.gandiva
-"
-
-export ARROW_TEST_DATA=/arrow/testing/data
-
-# Run pyarrow tests
-pip install -q -r /arrow/python/requirements-test.txt
-pytest -v --pyargs pyarrow
+/opt/python/cp37-cp37m/bin/pip install cmake ninja
+ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake
+ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja
+strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*