You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2020/04/20 19:22:03 UTC
[arrow] 21/28: ARROW-8517: [Release] Update Crossbow release
verification tasks for 0.17.0 RC0
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 84e3f19fde3c4a9ce688d093bc76eddc17d8c0c2
Author: Neal Richardson <ne...@gmail.com>
AuthorDate: Sun Apr 19 15:54:50 2020 -0500
ARROW-8517: [Release] Update Crossbow release verification tasks for 0.17.0 RC0
Closes #6961 from nealrichardson/verify-rc-0.17.0-0
Lead-authored-by: Neal Richardson <ne...@gmail.com>
Co-authored-by: Sutou Kouhei <ko...@clear-code.com>
Co-authored-by: Wes McKinney <we...@apache.org>
Signed-off-by: Wes McKinney <we...@apache.org>
---
dev/release/verify-release-candidate.sh | 38 ++++++++++++++++-----------
dev/tasks/tasks.yml | 25 ++++++++++++++++--
dev/tasks/verify-rc/github.nix.yml | 16 +++++++----
dev/tasks/verify-rc/github.windows.source.yml | 2 +-
dev/tasks/verify-rc/github.windows.wheels.yml | 2 +-
5 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 7961f2d..a6de4ff 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -127,7 +127,9 @@ test_binary() {
local download_dir=binaries
mkdir -p ${download_dir}
- python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir}
+ python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
+ --dest=${download_dir}
+
verify_dir_artifact_signatures ${download_dir}
}
@@ -646,6 +648,7 @@ test_linux_wheels() {
local env=_verify_wheel-${py_arch}
conda create -yq -n ${env} python=${py_arch//[mu]/}
conda activate ${env}
+ pip install -U pip
for ml_spec in ${manylinuxes}; do
# check the mandatory and optional imports
@@ -668,6 +671,7 @@ test_macos_wheels() {
local env=_verify_wheel-${py_arch}
conda create -yq -n ${env} python=${py_arch//m/}
conda activate ${env}
+ pip install -U pip
macos_suffix=macosx
case "${py_arch}" in
@@ -703,9 +707,6 @@ test_wheels() {
local filter_regex=.*manylinux.*
fi
- conda create -yq -n py3-base python=3.7
- conda activate py3-base
-
python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
--regex=${filter_regex} \
--dest=${download_dir}
@@ -726,9 +727,17 @@ test_wheels() {
# By default test all functionalities.
# To deactivate one test, deactivate the test and all of its dependents
# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
+
if [ "${ARTIFACT}" == "source" ]; then
- TEST_SOURCE=1
+ : ${TEST_SOURCE:=1}
+elif [ "${ARTIFACT}" == "wheels" ]; then
+ TEST_WHEELS=1
+else
+ TEST_BINARY_DISTRIBUTIONS=1
fi
+: ${TEST_SOURCE:=0}
+: ${TEST_WHEELS:=0}
+: ${TEST_BINARY_DISTRIBUTIONS:=0}
: ${TEST_DEFAULT:=1}
: ${TEST_JAVA:=${TEST_DEFAULT}}
@@ -741,9 +750,14 @@ fi
: ${TEST_GO:=${TEST_DEFAULT}}
: ${TEST_RUST:=${TEST_DEFAULT}}
: ${TEST_INTEGRATION:=${TEST_DEFAULT}}
-: ${TEST_BINARY:=${TEST_DEFAULT}}
-: ${TEST_APT:=${TEST_DEFAULT}}
-: ${TEST_YUM:=${TEST_DEFAULT}}
+if [ ${TEST_BINARY_DISTRIBUTIONS} -gt 0 ]; then
+ TEST_BINARY_DISTRIBUTIONS_DEFAULT=${TEST_DEFAULT}
+else
+ TEST_BINARY_DISTRIBUTIONS_DEFAULT=0
+fi
+: ${TEST_BINARY:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
+: ${TEST_APT:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
+: ${TEST_YUM:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
# For selective Integration testing, set TEST_DEFAULT=0 TEST_INTEGRATION_X=1 TEST_INTEGRATION_Y=1
: ${TEST_INTEGRATION_CPP:=${TEST_INTEGRATION}}
@@ -759,13 +773,7 @@ TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION_JS}))
TEST_GO=$((${TEST_GO} + ${TEST_INTEGRATION_GO}))
TEST_INTEGRATION=$((${TEST_INTEGRATION} + ${TEST_INTEGRATION_CPP} + ${TEST_INTEGRATION_JAVA} + ${TEST_INTEGRATION_JS} + ${TEST_INTEGRATION_GO}))
-if [ "${ARTIFACT}" == "wheels" ]; then
- TEST_WHEELS=1
-else
- TEST_WHEELS=0
-fi
-
-NEED_MINICONDA=$((${TEST_CPP} + ${TEST_WHEELS} + ${TEST_INTEGRATION}))
+NEED_MINICONDA=$((${TEST_CPP} + ${TEST_WHEELS} + ${TEST_BINARY} + ${TEST_INTEGRATION}))
: ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz}
case "${TEST_ARCHIVE}" in
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index dc2215f..e4f3331 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -76,6 +76,9 @@ groups:
verify-rc:
- verify-rc-*
+ verify-rc-binaries:
+ - verify-rc-binaries-*
+
verify-rc-wheels:
- verify-rc-wheels-*
@@ -1420,14 +1423,32 @@ tasks:
########################### Release verification ############################
- verify-rc-binaries:
+ verify-rc-binaries-binary:
ci: github
platform: linux
template: verify-rc/github.nix.yml
params:
os: "ubuntu"
artifact: "binaries"
- flag: ""
+ flag: "TEST_BINARY=1"
+
+ verify-rc-binaries-apt:
+ ci: github
+ platform: linux
+ template: verify-rc/github.nix.yml
+ params:
+ os: "ubuntu"
+ artifact: "binaries"
+ flag: "TEST_APT=1"
+
+ verify-rc-binaries-yum:
+ ci: github
+ platform: linux
+ template: verify-rc/github.nix.yml
+ params:
+ os: "ubuntu"
+ artifact: "binaries"
+ flag: "TEST_YUM=1"
verify-rc-wheels-linux:
ci: github
diff --git a/dev/tasks/verify-rc/github.nix.yml b/dev/tasks/verify-rc/github.nix.yml
index df26a7f..09455ee 100644
--- a/dev/tasks/verify-rc/github.nix.yml
+++ b/dev/tasks/verify-rc/github.nix.yml
@@ -45,12 +45,17 @@ jobs:
{{ flag }}
if [ $(uname) = "Darwin" ]; then
- brew install llvm@7 thrift glib
+ brew update
+ brew bundle --file=arrow/cpp/Brewfile
+ brew bundle --file=arrow/c_glib/Brewfile
+ if [ "$TEST_PYTHON" = "1" ]; then
+ # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package
+ export MACOSX_DEPLOYMENT_TARGET=10.9
+ fi
else
# TODO: don't require removing newer llvms
- sudo apt-get --purge remove -y llvm-8 llvm-9 clang-8 clang-9
+ sudo apt-get --purge remove -y llvm-9 clang-9
sudo apt-get install -y \
- llvm-7 clang-7 \
wget curl libboost-all-dev jq \
autoconf-archive gtk-doc-tools libgirepository1.0-dev flex bison
if [ "$TEST_JAVA" = "1" ]; then
@@ -64,8 +69,9 @@ jobs:
fi
if [ "$TEST_RUBY" = "1" ]; then
ruby --version
+ sudo gem install bundler
fi
fi
- # TODO: put 0.16.0 2 in some separate file?
+ # TODO: put version and rc number in some separate file?
# If you edit the versions, be sure to edit the other workflow files in this directory too
- TEST_DEFAULT=0 {{ flag }} arrow/dev/release/verify-release-candidate.sh {{ artifact }} 0.16.0 2
+ TEST_DEFAULT=0 {{ flag }} arrow/dev/release/verify-release-candidate.sh {{ artifact }} 0.17.0 0
diff --git a/dev/tasks/verify-rc/github.windows.source.yml b/dev/tasks/verify-rc/github.windows.source.yml
index ac343ad..d236bb0 100644
--- a/dev/tasks/verify-rc/github.windows.source.yml
+++ b/dev/tasks/verify-rc/github.windows.source.yml
@@ -46,4 +46,4 @@ jobs:
choco install wget
cd arrow
# If you edit the versions, be sure to edit the other workflow files in this directory too
- dev/release/verify-release-candidate.bat 0.16.0 2
+ dev/release/verify-release-candidate.bat 0.17.0 0
diff --git a/dev/tasks/verify-rc/github.windows.wheels.yml b/dev/tasks/verify-rc/github.windows.wheels.yml
index 5ae83f2..082c2aa 100644
--- a/dev/tasks/verify-rc/github.windows.wheels.yml
+++ b/dev/tasks/verify-rc/github.windows.wheels.yml
@@ -45,4 +45,4 @@ jobs:
choco install wget
cd arrow
# If you edit the versions, be sure to edit the other workflow files in this directory too
- dev/release/verify-release-candidate-wheels.bat 0.16.0 2
+ dev/release/verify-release-candidate-wheels.bat 0.17.0 0