You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2021/03/15 10:54:02 UTC
[arrow] branch master updated: ARROW-10349: [Python] Build and
publish aarch64 wheels
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f7b1439 ARROW-10349: [Python] Build and publish aarch64 wheels
f7b1439 is described below
commit f7b143959ec77a79a3e11bcb83e24eaf0d03697c
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Mon Mar 15 11:52:48 2021 +0100
ARROW-10349: [Python] Build and publish aarch64 wheels
This should work in theory, I'm building the image to see how well does vcpkg support arm packages.
```bash
ARCH=arm64v8 archery docker build python-wheel-manylinux-2014
```
Closes #9285 from kszucs/manylinux-arm64
Lead-authored-by: Krisztián Szűcs <sz...@gmail.com>
Co-authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Krisztián Szűcs <sz...@gmail.com>
---
.env | 6 +-
ci/docker/python-wheel-manylinux-201x.dockerfile | 15 ++-
.../docker/python-wheel-manylinux-test.dockerfile | 11 +-
ci/scripts/python_wheel_manylinux_build.sh | 11 +-
ci/scripts/python_wheel_manylinux_test.sh | 36 +++++--
.../vcpkg/arm64-linux-static-debug.cmake | 10 +-
.../vcpkg/arm64-linux-static-release.cmake | 10 +-
ci/vcpkg/ports.patch | 15 +++
dev/archery/archery/bot.py | 2 +-
dev/archery/archery/docker.py | 15 +++
.../tests/fixtures/crossbow-success-message.md | 4 +-
dev/release/verify-release-candidate.sh | 1 -
dev/tasks/linux-packages/Rakefile | 1 -
dev/tasks/linux-packages/travis.linux.arm64.yml | 50 ++++++++-
.../{github.linux.yml => github.linux.amd64.yml} | 30 +++---
dev/tasks/python-wheels/travis.linux.arm64.yml | 91 +++++++++++++++++
dev/tasks/tasks.yml | 113 +++++++++++----------
docker-compose.yml | 41 ++++++--
python/requirements-wheel-build.txt | 5 +-
python/requirements-wheel-test.txt | 9 +-
20 files changed, 355 insertions(+), 121 deletions(-)
diff --git a/.env b/.env
index 5a13886..535a765 100644
--- a/.env
+++ b/.env
@@ -33,9 +33,13 @@ COMPOSE_DOCKER_CLI_BUILD=1
DOCKER_BUILDKIT=1
BUILDKIT_INLINE_CACHE=1
+# different architecture notations
+ARCH=amd64
+ARCH_ALIAS=x86_64
+ARCH_SHORT_ALIAS=x64
+
ULIMIT_CORE=-1
REPO=apache/arrow-dev
-ARCH=amd64
CUDA=9.1
DEBIAN=10
UBUNTU=18.04
diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index 2bdb7a9..d9b3826 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -18,11 +18,14 @@
ARG base
FROM ${base}
+ARG arch_alias
+ARG arch_short_alias
+
RUN yum install -y git flex curl autoconf zip wget
# Install CMake
-ARG cmake=3.19.2
-RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz -O - | \
+ARG cmake=3.19.3
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
tar -xzf - --directory /usr/local --strip-components=1
# Install Ninja
@@ -51,7 +54,7 @@ RUN mkdir /tmp/ccache && \
ARG vcpkg
RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
git -C /opt/vcpkg checkout ${vcpkg} && \
- /opt/vcpkg/bootstrap-vcpkg.sh --useSystemBinaries --disableMetrics && \
+ /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \
ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg
# Patch ports files as needed
@@ -62,12 +65,16 @@ ARG build_type=release
ENV CMAKE_BUILD_TYPE=${build_type} \
VCPKG_FORCE_SYSTEM_BINARIES=1 \
VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
- VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type} \
+ VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
VCPKG_FEATURE_FLAGS=-manifests
+# Need to install the boost-build prior installing the boost packages, otherwise
+# vcpkg will raise an error.
# TODO(kszucs): factor out the package enumeration to a text file and reuse it
# from the windows image and potentially in a future macos wheel build
RUN vcpkg install --clean-after-build \
+ boost-build:${arch_short_alias}-linux && \
+ vcpkg install --clean-after-build \
abseil \
aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
boost-filesystem \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from b/ci/docker/python-wheel-manylinux-test.dockerfile
similarity index 71%
copy from dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
copy to ci/docker/python-wheel-manylinux-test.dockerfile
index 0a2ae2b..55c27d1 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
+++ b/ci/docker/python-wheel-manylinux-test.dockerfile
@@ -15,4 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-arm64v8/centos:7
+ARG arch
+ARG python
+FROM ${arch}/python:${python}
+
+# RUN pip install --upgrade pip
+
+# pandas doesn't provide wheel for aarch64 yet, so cache the compiled
+# test dependencies in a docker image
+COPY python/requirements-wheel-test.txt /arrow/python/
+RUN pip install -r /arrow/python/requirements-wheel-test.txt
diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
index 6a11461..83aa623 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -66,8 +66,10 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
: ${ARROW_WITH_ZLIB:=ON}
: ${ARROW_WITH_ZSTD:=ON}
: ${CMAKE_BUILD_TYPE:=release}
+: ${CMAKE_UNITY_BUILD:=ON}
: ${CMAKE_GENERATOR:=Ninja}
: ${VCPKG_FEATURE_FLAGS:=-manifests}
+: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}
mkdir /tmp/arrow-build
pushd /tmp/arrow-build
@@ -103,10 +105,10 @@ cmake \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
- -DCMAKE_UNITY_BUILD=ON \
+ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
-DOPENSSL_USE_STATIC_LIBS=ON \
-DVCPKG_MANIFEST_MODE=OFF \
- -DVCPKG_TARGET_TRIPLET=x64-linux-static-${CMAKE_BUILD_TYPE} \
+ -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \
-G ${CMAKE_GENERATOR} \
/arrow/cpp
cmake --build . --target install
@@ -135,8 +137,5 @@ pushd /arrow/python
python setup.py bdist_wheel
echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ==="
-auditwheel repair \
- --plat "manylinux${MANYLINUX_VERSION}_x86_64" \
- -L . dist/pyarrow-*.whl \
- -w repaired_wheels
+auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels
popd
diff --git a/ci/scripts/python_wheel_manylinux_test.sh b/ci/scripts/python_wheel_manylinux_test.sh
index d603f7c..2198774 100755
--- a/ci/scripts/python_wheel_manylinux_test.sh
+++ b/ci/scripts/python_wheel_manylinux_test.sh
@@ -17,7 +17,23 @@
# specific language governing permissions and limitations
# under the License.
-set -ex
+set -e
+set -x
+set -o pipefail
+
+case $# in
+ 1) KIND="$1"
+ case $KIND in
+ imports|unittests) ;;
+ *) echo "Invalid argument: '${KIND}', valid options are 'imports', 'unittests'"
+ exit 1
+ ;;
+ esac
+ ;;
+ *) echo "Usage: $0 imports|unittests"
+ exit 1
+ ;;
+esac
export PYARROW_TEST_CYTHON=OFF
export PYARROW_TEST_DATASET=ON
@@ -37,8 +53,9 @@ export PARQUET_TEST_DATA=/arrow/submodules/parquet-testing/data
# Install the built wheels
pip install /arrow/python/repaired_wheels/*.whl
-# Test that the modules are importable
-python -c "
+if [ "${KIND}" == "imports" ]; then
+ # Test that the modules are importable
+ python -c "
import pyarrow
import pyarrow._hdfs
import pyarrow._s3fs
@@ -49,11 +66,8 @@ import pyarrow.fs
import pyarrow.json
import pyarrow.orc
import pyarrow.parquet
-import pyarrow.plasma
-"
-
-# Install testing dependencies
-pip install -r /arrow/python/requirements-wheel-test.txt
-
-# Execute unittest
-pytest -v -r s --pyargs pyarrow
+import pyarrow.plasma"
+elif [ "${KIND}" == "unittests" ]; then
+ # Execute unittest, test dependencies must be installed
+ pytest -r s --pyargs pyarrow
+fi
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from b/ci/vcpkg/arm64-linux-static-debug.cmake
similarity index 70%
copy from dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
copy to ci/vcpkg/arm64-linux-static-debug.cmake
index 0a2ae2b..5d77b8d 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
+++ b/ci/vcpkg/arm64-linux-static-debug.cmake
@@ -15,4 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-arm64v8/centos:7
+set(VCPKG_TARGET_ARCHITECTURE arm64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+set(VCPKG_CMAKE_SYSTEM_NAME Linux)
+set(VCPKG_BUILD_TYPE debug)
+
+if(NOT CMAKE_HOST_SYSTEM_PROCESSOR)
+ execute_process(COMMAND "uname" "-m" OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from b/ci/vcpkg/arm64-linux-static-release.cmake
similarity index 70%
rename from dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
rename to ci/vcpkg/arm64-linux-static-release.cmake
index 0a2ae2b..ebe5bc3 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7-aarch64/from
+++ b/ci/vcpkg/arm64-linux-static-release.cmake
@@ -15,4 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-arm64v8/centos:7
+set(VCPKG_TARGET_ARCHITECTURE arm64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+set(VCPKG_CMAKE_SYSTEM_NAME Linux)
+set(VCPKG_BUILD_TYPE release)
+
+if(NOT CMAKE_HOST_SYSTEM_PROCESSOR)
+ execute_process(COMMAND "uname" "-m" OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 106bf72..b1f4f09 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -1,3 +1,18 @@
+diff --git a/ports/aws-c-common/portfile.cmake b/ports/aws-c-common/portfile.cmake
+index f3704ef05..3af543058 100644
+--- a/ports/aws-c-common/portfile.cmake
++++ b/ports/aws-c-common/portfile.cmake
+@@ -1,8 +1,8 @@
+ vcpkg_from_github(
+ OUT_SOURCE_PATH SOURCE_PATH
+ REPO awslabs/aws-c-common
+- REF 4a21a1c0757083a16497fea27886f5f20ccdf334 # v0.4.56
+- SHA512 68898a8ac15d5490f45676eabfbe0df9e45370a74c543a28909fd0d85fed48dfcf4bcd6ea2d01d1a036dd352e2e4e0b08c48c63ab2a2b477fe150b46a827136e
++ REF 13adef72b7813ec878817c6d50a7a3f241015d8a # v0.4.57
++ SHA512 28256522ac6af544d7464e3e7dcd4dc802ae2b09728bf8f167f86a6487bb756d0cad5eb4a2480610b2967b9c24c4a7f70621894517aa2828ffdeb0479453803b
+ HEAD_REF master
+ PATCHES
+ disable-error-4068.patch # This patch fixes dependency port compilation failure
diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
index 6e18aecd0..2ccecf33c 100644
--- a/ports/curl/portfile.cmake
diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py
index d81be5c..54a57f2 100644
--- a/dev/archery/archery/bot.py
+++ b/dev/archery/archery/bot.py
@@ -107,7 +107,7 @@ class CrossbowCommentFormatter:
),
'travis': _markdown_badge.format(
title='TravisCI',
- url='https://travis-ci.org/{repo}/branches',
+ url='https://travis-ci.com/{repo}/branches',
badge='https://img.shields.io/travis/{repo}/{branch}.svg'
),
'circle': _markdown_badge.format(
diff --git a/dev/archery/archery/docker.py b/dev/archery/archery/docker.py
index c24170c..17d4c71 100644
--- a/dev/archery/archery/docker.py
+++ b/dev/archery/archery/docker.py
@@ -80,6 +80,21 @@ class ComposeConfig:
# override the defaults passed as parameters
self.env.update(self.params)
+ # translate docker's architecture notation to a more widely used one
+ arch = self.env.get('ARCH', 'amd64')
+ arch_aliases = {
+ 'amd64': 'x86_64',
+ 'arm64v8': 'aarch64',
+ 's390x': 's390x'
+ }
+ arch_short_aliases = {
+ 'amd64': 'x64',
+ 'arm64v8': 'arm64',
+ 's390x': 's390x'
+ }
+ self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch)
+ self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch)
+
def _read_config(self, config_path, compose_bin):
"""
Validate and read the docker-compose.yml
diff --git a/dev/archery/archery/tests/fixtures/crossbow-success-message.md b/dev/archery/archery/tests/fixtures/crossbow-success-message.md
index e45c146..1582521 100644
--- a/dev/archery/archery/tests/fixtures/crossbow-success-message.md
+++ b/dev/archery/archery/tests/fixtures/crossbow-success-message.md
@@ -5,6 +5,6 @@ Submitted crossbow builds: [{repo} @ {branch}](https://github.com/{repo}/branche
|Task|Status|
|----|------|
|docker-cpp-cmake32|[](https://circleci.com/gh/{repo}/tree/{branch}-circle-docker-cpp-cmake32)|
-|wheel-osx-cp36m|[](https://travis-ci.org/{repo}/branches)|
-|wheel-osx-cp37m|[](https://travis-ci.org/{repo}/branches)|
+|wheel-osx-cp36m|[](https://travis-ci.com/{repo}/branches)|
+|wheel-osx-cp37m|[](https://travis-ci.com/{repo}/branches)|
|wheel-win-cp36m|[](https://ci.appveyor.com/project/{repo}/history)|
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 3d39f4e..d208a38 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -166,7 +166,6 @@ test_apt() {
test_yum() {
for target in "centos:7" \
- "arm64v8/centos:7" \
"centos:8" \
"arm64v8/centos:8"; do
case "${target}" in
diff --git a/dev/tasks/linux-packages/Rakefile b/dev/tasks/linux-packages/Rakefile
index 36bb0f8..84ad76f 100644
--- a/dev/tasks/linux-packages/Rakefile
+++ b/dev/tasks/linux-packages/Rakefile
@@ -202,7 +202,6 @@ class LocalBinaryTask < BinaryTask
# because they require some setups on host.
[
"centos-7",
- # "centos-7-aarch64",
"centos-8",
# "centos-8-aarch64",
]
diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml
index 4221db2..0b83520 100644
--- a/dev/tasks/linux-packages/travis.linux.arm64.yml
+++ b/dev/tasks/linux-packages/travis.linux.arm64.yml
@@ -15,16 +15,19 @@
# specific language governing permissions and limitations
# under the License.
+arch: arm64-graviton2
+virt: vm
os: linux
-arch: arm64
-dist: bionic
+dist: focal
+group: edge
language: minimal
addons:
apt:
packages:
- apt-utils
- - createrepo
+ # We need Ubuntu 20.10 or later
+ # - createrepo_c
- devscripts
- gpg
- libgit2-dev
@@ -32,6 +35,22 @@ addons:
- rake
- rpm
+ # To build createrepo_c from source.
+ # We can remove them when we can install createrepo_c package
+ - cmake
+ - libbz2-dev
+ - libcurl4-openssl-dev
+ - libglib2.0-dev
+ - liblzma-dev
+ - libmagic-dev
+ - librpm-dev
+ - libsqlite3-dev
+ - libssl-dev
+ - libxml2-dev
+ - libzstd-dev
+ - pkg-config
+ - zlib1g-dev
+
services:
- docker
@@ -51,6 +70,24 @@ before_script:
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- git -C arrow checkout FETCH_HEAD
+ # Build createrepo_c from source.
+ # We can remove them when we can install createrepo_c package
+ - git clone --depth 1 https://github.com/rpm-software-management/createrepo_c.git
+ - pushd createrepo_c
+ - |
+ /usr/bin/cmake \
+ -DCMAKE_INSTALL_PREFIX=/usr \
+ -DENABLE_BASHCOMP=OFF \
+ -DENABLE_DRPM=OFF \
+ -DENABLE_PYTHON=OFF \
+ -DWITH_LIBMODULEMD=OFF \
+ -DWITH_ZCHUNK=OFF \
+ .
+ - make -j$(nproc)
+ - sudo make install
+ - popd
+ - rm -rf createrepo_c
+
script:
# Build packages
- pushd arrow/dev/tasks/linux-packages
@@ -99,12 +136,15 @@ script:
- gpg --export --armor test@example.com > arrow/dev/tasks/linux-packages/KEYS
- pushd arrow/dev/tasks/linux-packages
- |
- rake --trace {{ task_namespace }}:test GPG_KEY_ID=${GPG_KEY_ID}
+ rake --trace {{ task_namespace }}:test \
+ CREATEREPO=createrepo_c \
+ GPG_KEY_ID=${GPG_KEY_ID}
- rm -rf {{ task_namespace }}/repositories
- popd
after_success:
- - sudo -H pip3 install --requirement arrow/dev/tasks/requirements-crossbow.txt
+ - sudo -H pip3 install --upgrade pip
+ - sudo -H pip3 install -r arrow/dev/tasks/requirements-crossbow.txt pygit2==1.0
- |
python3 arrow/dev/tasks/crossbow.py \
--queue-path . \
diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.amd64.yml
similarity index 76%
rename from dev/tasks/python-wheels/github.linux.yml
rename to dev/tasks/python-wheels/github.linux.amd64.yml
index 3f2393c..779aa54 100644
--- a/dev/tasks/python-wheels/github.linux.yml
+++ b/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -29,18 +29,9 @@ jobs:
name: "Build wheel for Manylinux {{ manylinux_version }}"
runs-on: ubuntu-latest
env:
- # archery uses this environment variable
+ # archery uses these environment variables
+ ARCH: amd64
PYTHON: {{ python_version }}
- # this is a private repository at the moment (mostly because of licensing
- # consideration of windows images with visual studio), but anyone can
- # recreate the image by manually building it via:
- # `archery build python-wheel-manylinux-2010|2014`
- # note that we don't run docker build since there wouldn't be a cache hit
- # and rebuilding the dependencies takes a fair amount of time
- # (in case of linux multi-host caching works much better with buildkit)
- REPO: ghcr.io/ursacomputing/arrow
- # prefer the docker cli over docker-compose
- ARCHERY_USE_DOCKER_CLI: 1
steps:
############################ Checkout ###################################
@@ -60,9 +51,8 @@ jobs:
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
with:
- registry: ghcr.io
- username: {{ '${{ github.repository_owner }}' }}
- password: {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }}
+ username: {{ '${{ secrets.DOCKERHUB_USER }}' }}
+ password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }}
############################ Archery Installation #######################
- name: Set up Python
@@ -75,11 +65,13 @@ jobs:
############################ Build & Test ###############################
- name: Build wheel
shell: bash
- run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+ run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
# TODO(kszucs): auditwheel show
- name: Test wheel
shell: bash
- run: archery docker run python-wheel-manylinux-test
+ run: |
+ archery docker run python-wheel-manylinux-test-imports
+ archery docker run python-wheel-manylinux-test-unittests
############################ Artifact Uploading #########################
- name: Upload artifacts
@@ -96,6 +88,12 @@ jobs:
CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }}
{% if arrow.branch == 'master' %}
+ - name: Push the docker image
+ shell: bash
+ run: |
+ archery docker push python-wheel-manylinux-{{ manylinux_version }}
+ archery docker push python-wheel-manylinux-test-unittests
+
- name: Upload to gemfury
shell: bash
run: |
diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml
new file mode 100644
index 0000000..01ae487
--- /dev/null
+++ b/dev/tasks/python-wheels/travis.linux.arm64.yml
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+arch: arm64-graviton2
+virt: vm
+os: linux
+dist: focal
+group: edge
+language: minimal
+
+addons:
+ apt:
+ packages:
+ - libgit2-dev
+ - python3-pip
+
+services:
+ - docker
+
+# don't build twice
+if: tag IS blank
+
+env:
+ global:
+ - BUILD_REF={{ arrow.head }}
+ - TRAVIS_TAG={{ task.tag }}
+ # archery uses these environment variables
+ - ARCH=arm64v8
+ - PYTHON={{ python_version }}
+
+before_script:
+ - git clone --no-checkout {{ arrow.remote }} arrow
+ - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+ - git -C arrow checkout FETCH_HEAD
+ - git -C arrow submodule update --init --recursive
+
+script:
+ # Install Archery and Crossbow dependencies
+ - sudo -H pip3 install --upgrade pip
+ - sudo -H pip3 install docker-compose
+ - sudo -H pip3 install -e arrow/dev/archery[docker]
+ - sudo -H pip3 install -r arrow/dev/tasks/requirements-crossbow.txt pygit2==1.0
+
+ # Docker Login:
+ - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}" --password-stdin
+
+ # Build and Test packages
+
+ # output something every minutes to prevent travis from killing the build
+ - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done &
+ - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+ - archery docker run python-wheel-manylinux-test-imports
+ - archery docker run python-wheel-manylinux-test-unittests
+ - kill %1
+
+after_success:
+ # Upload wheel as github artifact
+ - |
+ python3 arrow/dev/tasks/crossbow.py \
+ --queue-path . \
+ --queue-remote {{ queue_remote_url }} \
+ upload-artifacts \
+ --pattern "arrow/python/repaired_wheels/*.whl" \
+ --sha {{ task.branch }} \
+ --tag {{ task.tag }}
+
+ {% if arrow.branch == 'master' %}
+ # Push the docker image to dockerhub
+ - archery docker push python-wheel-manylinux-{{ manylinux_version }}
+ - archery docker push python-wheel-manylinux-test-unittests
+ # Upload as nightly wheel to gemfury
+ - |
+ WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl)
+ curl \
+ -F "package=@${WHEEL_PATH}" \
+ "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/"
+ {% endif %}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index c41fa69..f8d6efd 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -121,9 +121,9 @@ groups:
######################## Tasks to run regularly #############################
nightly:
- - debian-*-amd64
- - ubuntu-*-amd64
- - centos-*-amd64
+ - debian-*
+ - ubuntu-*
+ - centos-*
- conda-*
- gandiva-*
# List the homebrews explicitly because we don't care about running homebrew-cpp-autobrew
@@ -345,78 +345,118 @@ tasks:
############################## Wheel Linux ##################################
- wheel-manylinux2010-cp36m:
+ # AMD64
+
+ wheel-manylinux2010-cp36m-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.6
manylinux_version: 2010
artifacts:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl
- wheel-manylinux2010-cp37m:
+ wheel-manylinux2010-cp37m-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.7
manylinux_version: 2010
artifacts:
- pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl
- wheel-manylinux2010-cp38:
+ wheel-manylinux2010-cp38-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.8
manylinux_version: 2010
artifacts:
- pyarrow-{no_rc_version}-cp38-cp38-manylinux2010_x86_64.whl
- wheel-manylinux2010-cp39:
+ wheel-manylinux2010-cp39-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.9
manylinux_version: 2010
artifacts:
- pyarrow-{no_rc_version}-cp39-cp39-manylinux2010_x86_64.whl
- wheel-manylinux2014-cp36m:
+ wheel-manylinux2014-cp36m-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.6
manylinux_version: 2014
artifacts:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux2014_x86_64.whl
- wheel-manylinux2014-cp37m:
+ wheel-manylinux2014-cp37m-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.7
manylinux_version: 2014
artifacts:
- pyarrow-{no_rc_version}-cp37-cp37m-manylinux2014_x86_64.whl
- wheel-manylinux2014-cp38:
+ wheel-manylinux2014-cp38-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.8
manylinux_version: 2014
artifacts:
- pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_x86_64.whl
- wheel-manylinux2014-cp39:
+ wheel-manylinux2014-cp39-amd64:
ci: github
- template: python-wheels/github.linux.yml
+ template: python-wheels/github.linux.amd64.yml
params:
python_version: 3.9
manylinux_version: 2014
artifacts:
- pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_x86_64.whl
+ # ARM64v8
+
+ wheel-manylinux2014-cp36m-arm64:
+ ci: travis
+ template: python-wheels/travis.linux.arm64.yml
+ params:
+ python_version: 3.6
+ manylinux_version: 2014
+ artifacts:
+ - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2014_aarch64.whl
+
+ wheel-manylinux2014-cp37m-arm64:
+ ci: travis
+ template: python-wheels/travis.linux.arm64.yml
+ params:
+ python_version: 3.7
+ manylinux_version: 2014
+ artifacts:
+ - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2014_aarch64.whl
+
+ wheel-manylinux2014-cp38-arm64:
+ ci: travis
+ template: python-wheels/travis.linux.arm64.yml
+ params:
+ python_version: 3.8
+ manylinux_version: 2014
+ artifacts:
+ - pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_aarch64.whl
+
+ wheel-manylinux2014-cp39-arm64:
+ ci: travis
+ template: python-wheels/travis.linux.arm64.yml
+ params:
+ python_version: 3.9
+ manylinux_version: 2014
+ artifacts:
+ - pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_aarch64.whl
+
############################## Wheel OSX ####################################
wheel-osx-mavericks-cp36m:
@@ -1030,41 +1070,6 @@ tasks:
- plasma-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- plasma-store-server-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- centos-7-aarch64:
- ci: travis
- template: linux-packages/travis.linux.arm64.yml
- params:
- target: "centos-7-aarch64"
- task_namespace: yum
- upload_extensions:
- - .rpm
- artifacts:
- - arrow-{no_rc_version}-1.el7.src.rpm
- - arrow-dataset-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-dataset-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-dataset-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-dataset-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-dataset-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-debuginfo-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-python-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - arrow-python-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - parquet-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - parquet-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - parquet-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - parquet-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - parquet-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-glib-devel-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-glib-doc-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-glib-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-libs-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
- - plasma-store-server-{no_rc_version}-1.el7.[a-z0-9_]+.rpm
-
centos-8-amd64:
ci: github
template: linux-packages/github.linux.amd64.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index c502038..b26348c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -122,7 +122,8 @@ x-hierarchy:
- postgres
- python-wheel-manylinux-2010
- python-wheel-manylinux-2014
- - python-wheel-manylinux-test
+ - python-wheel-manylinux-test-imports
+ - python-wheel-manylinux-test-unittests
- python-wheel-windows-vs2017
- python-wheel-windows-test
@@ -695,17 +696,20 @@ services:
# See available versions at:
# https://quay.io/repository/pypa/manylinux2010_x86_64?tab=tags
+ # only amd64 arch is supported
python-wheel-manylinux-2010:
- image: ${REPO}:python-${PYTHON}-wheel-manylinux-2010
+ image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010
build:
args:
- base: quay.io/pypa/manylinux2010_x86_64:2020-12-03-912b0de
+ arch_alias: ${ARCH_ALIAS}
+ arch_short_alias: ${ARCH_SHORT_ALIAS}
+ base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2020-12-03-912b0de
vcpkg: ${VCPKG}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
cache_from:
- - ${REPO}:python-${PYTHON}-wheel-manylinux-2010
+ - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010
environment:
<<: *ccache
MANYLINUX_VERSION: 2010
@@ -717,16 +721,18 @@ services:
# See available versions at:
# https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags
python-wheel-manylinux-2014:
- image: ${REPO}:python-${PYTHON}-wheel-manylinux-2014
+ image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014
build:
args:
- base: quay.io/pypa/manylinux2014_x86_64:2020-11-11-bc8ce45
+ arch_alias: ${ARCH_ALIAS}
+ arch_short_alias: ${ARCH_SHORT_ALIAS}
+ base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2020-11-11-bc8ce45
vcpkg: ${VCPKG}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
cache_from:
- - ${REPO}:python-${PYTHON}-wheel-manylinux-2014
+ - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014
environment:
<<: *ccache
MANYLINUX_VERSION: 2014
@@ -735,12 +741,27 @@ services:
- ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
command: /arrow/ci/scripts/python_wheel_manylinux_build.sh
- python-wheel-manylinux-test:
- image: python:${PYTHON}
+ python-wheel-manylinux-test-imports:
+ image: ${ARCH}/python:${PYTHON}
shm_size: 2G
volumes:
- .:/arrow:delegated
- command: /arrow/ci/scripts/python_wheel_manylinux_test.sh
+ command: /arrow/ci/scripts/python_wheel_manylinux_test.sh imports
+
+ python-wheel-manylinux-test-unittests:
+ image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
+ build:
+ args:
+ arch: ${ARCH}
+ python: ${PYTHON}
+ context: .
+ dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile
+ cache_from:
+ - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
+ shm_size: 2G
+ volumes:
+ - .:/arrow:delegated
+ command: /arrow/ci/scripts/python_wheel_manylinux_test.sh unittests
python-wheel-windows-vs2017:
image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017
diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt
index 457dea3..74a352e 100644
--- a/python/requirements-wheel-build.txt
+++ b/python/requirements-wheel-build.txt
@@ -1,5 +1,6 @@
cython>=0.29.11
-numpy==1.16.6; python_version < "3.9"
-numpy==1.19.4; python_version >= "3.9"
+numpy==1.16.6; python_version < "3.9" and platform_machine != "aarch64"
+numpy==1.19.4; python_version >= "3.9" or platform_machine == "aarch64"
+pandas<1.1.0; python_version < "3.8"
setuptools_scm
wheel
diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt
index 01aca24..7377f6b 100644
--- a/python/requirements-wheel-test.txt
+++ b/python/requirements-wheel-test.txt
@@ -1,10 +1,11 @@
cffi
cython
hypothesis
-numpy==1.19.4
-pandas<1.1.0; python_version < "3.8"
-pandas; python_version >= "3.8"
-pickle5; (python_version == "3.6" or python_version == "3.7") and sys_platform != 'win32'
+numpy==1.19.4; python_version != "3.9" or platform_machine != "aarch64"
+numpy==1.20.1; python_version == "3.9" and platform_machine == "aarch64"
+pandas<1.1.0; python_version < "3.8" and platform_machine != "aarch64"
+pandas; python_version >= "3.8" or platform_machine == "aarch64"
+pickle5; (python_version == "3.6" or python_version == "3.7") and sys_platform != "win32"
pytest
pytest-lazy-fixture
pytz