You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/10/11 05:18:16 UTC
[arrow] branch master updated: ARROW-17240: [CI][Release] Verify wheels in nightly CI (#14319)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cd1c01cfcb ARROW-17240: [CI][Release] Verify wheels in nightly CI (#14319)
cd1c01cfcb is described below
commit cd1c01cfcb4eed31707872ffb5a24be137bcff24
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Tue Oct 11 14:18:10 2022 +0900
ARROW-17240: [CI][Release] Verify wheels in nightly CI (#14319)
.deb and .rpm are already verified by the same verification scripts (dev/release/verify-apt.sh and dev/release/verify-yum.sh) but .whl aren't yet.
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
ci/docker/almalinux-8-verify-rc.dockerfile | 39 +-----------
dev/release/setup-rhel-rebuilds.sh | 5 ++
dev/release/verify-release-candidate.sh | 38 ++++++-----
dev/tasks/python-wheels/github.linux.amd64.yml | 77 ++++++++++++++++++++++-
dev/tasks/python-wheels/github.osx.amd64.yml | 27 +++++---
dev/tasks/python-wheels/github.osx.arm64.yml | 25 +++++---
dev/tasks/tasks.yml | 9 ---
dev/tasks/verify-rc/github.linux.amd64.docker.yml | 2 +-
8 files changed, 143 insertions(+), 79 deletions(-)
diff --git a/ci/docker/almalinux-8-verify-rc.dockerfile b/ci/docker/almalinux-8-verify-rc.dockerfile
index 94e8a1133d..e9544e6bec 100644
--- a/ci/docker/almalinux-8-verify-rc.dockerfile
+++ b/ci/docker/almalinux-8-verify-rc.dockerfile
@@ -18,40 +18,7 @@
ARG arch=amd64
FROM ${arch}/almalinux:8
-# A script to install dependencies required for release
-# verification Red Hat Enterprise Linux 8 clones in particular
-# on AlmaLinux 8 and Rocky Linux 8
-
-RUN dnf -y install 'dnf-command(config-manager)' && \
- dnf config-manager --set-enabled powertools && \
- dnf -y update && \
- dnf -y module disable nodejs && \
- dnf -y module enable nodejs:16 && \
- dnf -y module disable ruby && \
- dnf -y module enable ruby:2.7 && \
- dnf -y groupinstall "Development Tools" && \
- dnf -y install \
- cmake \
- git \
- gobject-introspection-devel \
- java-1.8.0-openjdk-devel \
- libcurl-devel \
- llvm-devel \
- llvm-toolset \
- maven \
- ncurses-devel \
- ninja-build \
- nodejs \
- openssl-devel \
- python38-devel \
- python38-pip \
- ruby-devel \
- sqlite-devel \
- wget \
- which && \
+COPY dev/release/setup-rhel-rebuilds.sh /
+RUN /setup-rhel-rebuilds.sh && \
+ rm /setup-rhel-rebuilds.sh && \
dnf -y clean all
-
-RUN python3 -m pip install -U pip && \
- alternatives --set python /usr/bin/python3
-
-RUN npm install -g yarn
diff --git a/dev/release/setup-rhel-rebuilds.sh b/dev/release/setup-rhel-rebuilds.sh
index 1fe3ba47dc..9cdc503215 100755
--- a/dev/release/setup-rhel-rebuilds.sh
+++ b/dev/release/setup-rhel-rebuilds.sh
@@ -21,6 +21,8 @@
# verification Red Hat Enterprise Linux 8 clones in particular
# on AlmaLinux 8 and Rocky Linux 8
+set -exu
+
dnf -y install 'dnf-command(config-manager)'
dnf config-manager --set-enabled powertools
dnf -y update
@@ -49,5 +51,8 @@ dnf -y install \
vala-devel \
wget \
which
+
npm install -g yarn
+
+python3 -m pip install -U pip
alternatives --set python /usr/bin/python3
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 133cdf9027..2184d408d7 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -1014,7 +1014,7 @@ test_linux_wheels() {
local arch="x86_64"
fi
- local python_versions="3.7m 3.8 3.9 3.10"
+ local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}"
local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
for python in ${python_versions}; do
@@ -1023,7 +1023,7 @@ test_linux_wheels() {
show_header "Testing Python ${pyver} wheel for platform ${platform}"
CONDA_ENV=wheel-${pyver}-${platform} PYTHON_VERSION=${pyver} maybe_setup_conda || exit 1
VENV_ENV=wheel-${pyver}-${platform} PYTHON_VERSION=${pyver} maybe_setup_virtualenv || continue
- pip install pyarrow-${VERSION}-cp${pyver/.}-cp${python/.}-${platform}.whl
+ pip install pyarrow-${TEST_PYARROW_VERSION:-${VERSION}}-cp${pyver/.}-cp${python/.}-${platform}.whl
INSTALL_PYARROW=OFF ARROW_GCS=${check_gcs} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
done
done
@@ -1087,23 +1087,31 @@ test_wheels() {
show_header "Downloading Python wheels"
maybe_setup_conda python || exit 1
- local download_dir=${ARROW_TMPDIR}/binaries
- mkdir -p ${download_dir}
-
- if [ "$(uname)" == "Darwin" ]; then
- local filter_regex=.*macosx.*
+ local wheels_dir=
+ if [ "${SOURCE_KIND}" = "local" ]; then
+ wheels_dir="${ARROW_SOURCE_DIR}/python/repaired_wheels"
else
- local filter_regex=.*manylinux.*
- fi
+ local download_dir=${ARROW_TMPDIR}/binaries
+ mkdir -p ${download_dir}
- ${PYTHON:-python3} $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
- --package_type python \
- --regex=${filter_regex} \
- --dest=${download_dir}
+ if [ "$(uname)" == "Darwin" ]; then
+ local filter_regex=.*macosx.*
+ else
+ local filter_regex=.*manylinux.*
+ fi
- verify_dir_artifact_signatures ${download_dir}
+ ${PYTHON:-python3} \
+ $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
+ --package_type python \
+ --regex=${filter_regex} \
+ --dest=${download_dir}
+
+ verify_dir_artifact_signatures ${download_dir}
+
+ wheels_dir=${download_dir}/python-rc/${VERSION}-rc${RC_NUMBER}
+ fi
- pushd ${download_dir}/python-rc/${VERSION}-rc${RC_NUMBER}
+ pushd ${wheels_dir}
if [ "$(uname)" == "Darwin" ]; then
test_macos_wheels
diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml
index 3b307e0b56..9266517072 100644
--- a/dev/tasks/python-wheels/github.linux.amd64.yml
+++ b/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -21,7 +21,7 @@
jobs:
build:
- name: "Build wheel for Manylinux {{ manylinux_version }}"
+ name: "Build wheel for manylinux {{ manylinux_version }}"
runs-on: ubuntu-latest
env:
# archery uses these environment variables
@@ -37,6 +37,11 @@ jobs:
shell: bash
run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+ - uses: actions/upload-artifact@v3
+ with:
+ name: wheel
+ path: arrow/python/repaired_wheels/*.whl
+
# TODO(kszucs): auditwheel show
- name: Test wheel
shell: bash
@@ -44,6 +49,76 @@ jobs:
archery docker run python-wheel-manylinux-test-imports
archery docker run python-wheel-manylinux-test-unittests
+ - name: Test wheel on AlmaLinux 8
+ shell: bash
+ if: |
+ '{{ python_version }}' == '3.8'
+ env:
+ ALMALINUX: "8"
+ run: |
+ archery docker run \
+ -e TEST_DEFAULT=0 \
+ -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
+ -e TEST_PYTHON_VERSIONS={{ python_version }} \
+ -e TEST_WHEELS=1 \
+ almalinux-verify-rc
+
+ - name: Test wheel on Ubuntu 18.04
+ shell: bash
+ if: |
+ '{{ python_version }}' == '3.8'
+ env:
+ UBUNTU: "18.04"
+ run: |
+ archery docker run \
+ -e TEST_DEFAULT=0 \
+ -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
+ -e TEST_PYTHON_VERSIONS={{ python_version }} \
+ -e TEST_WHEELS=1 \
+ ubuntu-verify-rc
+
+ - name: Test wheel on Ubuntu 18.04
+ shell: bash
+ if: |
+ '{{ python_version }}' == '3.8'
+ env:
+ UBUNTU: "18.04"
+ run: |
+ archery docker run \
+ -e TEST_DEFAULT=0 \
+ -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
+ -e TEST_PYTHON_VERSIONS={{ python_version }} \
+ -e TEST_WHEELS=1 \
+ ubuntu-verify-rc
+
+ - name: Test wheel on Ubuntu 20.04
+ shell: bash
+ if: |
+ '{{ python_version }}' == '3.8'
+ env:
+ UBUNTU: "20.04"
+ run: |
+ archery docker run \
+ -e TEST_DEFAULT=0 \
+ -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
+ -e TEST_PYTHON_VERSIONS={{ python_version }} \
+ -e TEST_WHEELS=1 \
+ ubuntu-verify-rc
+
+ - name: Test wheel on Ubuntu 22.04
+ shell: bash
+ if: |
+ '{{ python_version }}' == '3.10'
+ env:
+ UBUNTU: "22.04"
+ run: |
+ archery docker run \
+ -e TEST_DEFAULT=0 \
+ -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
+ -e TEST_PYTHON_VERSIONS={{ python_version }} \
+ -e TEST_WHEELS=1 \
+ ubuntu-verify-rc
+
{{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index 25722c9e9c..8044fc39fb 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -33,7 +33,7 @@ env:
jobs:
build:
- name: Build wheel for macOS
+ name: Build wheel for Python {{ python_version }} on macOS
runs-on: macos-latest
env:
VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
@@ -103,15 +103,28 @@ jobs:
pip install --upgrade pip wheel
PYTHON=python arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build
+ - uses: actions/upload-artifact@v3
+ with:
+ name: wheel
+ path: arrow/python/repaired_wheels/*.whl
+
- name: Test Wheel
shell: bash
+ env:
+ TEST_DEFAULT: "0"
+ TEST_PYARROW_VERSION: "{{ arrow.no_rc_version }}"
+ TEST_WHEEL: "1"
run: |
- $PYTHON -m venv test-env
- source test-env/bin/activate
- pip install --upgrade pip wheel
- pip install -r arrow/python/requirements-wheel-test.txt
- PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default
- arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
+ case "${PYTHON_VERSION}" in
+ 3.7)
+ python_version=${PYTHON_VERSION}m
+ ;;
+ *)
+ python_version=${PYTHON_VERSION}
+ ;;
+ esac
+ export TEST_PYTHON_VERSIONS=${python_version}
+ arrow/dev/release/verify-release-candidate.sh
{{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index a02661f492..a555b28713 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -39,7 +39,7 @@ env:
jobs:
build:
- name: Build wheel for OS X
+ name: Build wheel for Python {{ python_version }} on macOS
runs-on: ["self-hosted", "macOS", "arm64"]
steps:
- name: Cleanup
@@ -67,7 +67,7 @@ jobs:
env:
VCPKG_DEFAULT_TRIPLET: arm64-osx-static-release
run: |
- vcpkg install \
+ vcpkg install \
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=arrow/ci/vcpkg \
@@ -85,14 +85,14 @@ jobs:
$PYTHON -m venv build-arm64-env
source build-arm64-env/bin/activate
pip install --upgrade pip wheel
- arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
+ arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
{% if arch == "universal2" %}
- name: Install AMD64 Packages
env:
VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
run: |
- vcpkg install \
+ vcpkg install \
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
--x-manifest-root=arrow/ci/vcpkg \
@@ -127,6 +127,11 @@ jobs:
mv $fused_wheel arrow/python/repaired_wheels/${fused_wheel/x86_64/universal2}
{% endif %}
+ - uses: actions/upload-artifact@v3
+ with:
+ name: wheel
+ path: arrow/python/repaired_wheels/*.whl
+
- name: Test Wheel on ARM64
shell: bash
env:
@@ -164,12 +169,12 @@ jobs:
source crossbow-env/bin/activate
arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload]
arch -x86_64 archery crossbow \
- --queue-path $(pwd) \
- --queue-remote {{ queue_remote_url }} \
- upload-artifacts \
- --sha {{ task.branch }} \
- --tag {{ task.tag }} \
- "arrow/python/repaired_wheels/*.whl"
+ --queue-path $(pwd) \
+ --queue-remote {{ queue_remote_url }} \
+ upload-artifacts \
+ --sha {{ task.branch }} \
+ --tag {{ task.tag }} \
+ "arrow/python/repaired_wheels/*.whl"
env:
CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 1945a327ad..81a2c24bba 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1076,15 +1076,6 @@ tasks:
params:
github_runner: "macos-{{ macos_version }}"
target: "wheels"
-
- verify-rc-binaries-wheels-macos-{{ macos_version }}-amd64-conda:
- ci: github
- template: verify-rc/github.macos.amd64.yml
- params:
- env:
- USE_CONDA: 1
- github_runner: "macos-{{ macos_version }}"
- target: "wheels"
{% endfor %}
verify-rc-binaries-wheels-macos-11-arm64:
diff --git a/dev/tasks/verify-rc/github.linux.amd64.docker.yml b/dev/tasks/verify-rc/github.linux.amd64.docker.yml
index 15e0e597a4..65b30b5c8d 100644
--- a/dev/tasks/verify-rc/github.linux.amd64.docker.yml
+++ b/dev/tasks/verify-rc/github.linux.amd64.docker.yml
@@ -21,7 +21,7 @@
jobs:
test:
- name: "Verify release candidate {{ distro }} source"
+ name: "Verify release candidate {{ distro }} {{ target }}"
runs-on: ubuntu-latest
{% if env is defined %}
env: