You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/02/25 21:20:11 UTC
[arrow-julia] branch main updated: Add verification script (#292)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
The following commit(s) were added to refs/heads/main by this push:
new 614fce0 Add verification script (#292)
614fce0 is described below
commit 614fce0a5d7db8fee078be32690c5220848538e2
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sat Feb 26 06:20:08 2022 +0900
Add verification script (#292)
fix #288
dev/release/verify_rc.sh verifies RC.
CI jobs for our RC related scripts are also added.
---
.github/workflows/ci.yml | 30 ++++++-
dev/release/README.md | 24 +++++-
dev/release/release_rc.sh | 51 ++++++++----
dev/release/run_rat.sh | 1 +
dev/release/verify_rc.sh | 205 ++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 293 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b989566..b8a0a18 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -22,7 +22,7 @@ on:
branches: [main]
tags: ['*']
jobs:
- rat:
+ license:
name: Audit licenses
runs-on: ubuntu-latest
steps:
@@ -32,6 +32,34 @@ jobs:
python-version: '3.x'
- name: Run Release audit tool (Rat)
run: dev/release/run_rat.sh .
+ release:
+ name: Verify release - ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os:
+ - macos-latest
+ - ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Create
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ RELEASE_DEFAULT=0 dev/release/release_rc.sh 1
+ - uses: actions/cache@v1
+ with:
+ path: ~/.julia/artifacts
+ key: ${{ runner.os }}-release-${{ hashFiles('**/Project.toml') }}
+ restore-keys: |
+ ${{ runner.os }}-release-
+ - name: Verify
+ run: |
+ version=$(grep -o '^version = ".*"' "Project.toml" | \
+ sed -e 's/^version = "//g' \
+ -e 's/"$//g')
+ VERIFY_DEFAULT=0 dev/release/verify_rc.sh ${version} 1
test:
name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
runs-on: ${{ matrix.os }}
diff --git a/dev/release/README.md b/dev/release/README.md
index fc8c8c8..bd382fa 100644
--- a/dev/release/README.md
+++ b/dev/release/README.md
@@ -95,4 +95,26 @@ JuliaRegistrator will respond saying it has opened a pull request to the General
### Verify
-TODO
+We have a script to verify a RC.
+
+You must install the following commands to use the script:
+
+ * `curl`
+ * `gpg`
+ * `shasum` or `sha256sum`/`sha512sum`
+
+You don't need to install Julia. If there isn't Julia in system, the latest Julia is automatically installed only for verification.
+
+To verify a RC, run the following command line:
+
+```console
+$ dev/release/verify.sh ${VERSION} ${RC}
+```
+
+Here is an example to release 2.2.1 RC1:
+
+```console
+$ dev/release/verify.sh 2.2.1 1
+```
+
+If the verification is succeeded, `RC looks good!` is shown.
diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh
index 45314e2..bae1997 100755
--- a/dev/release/release_rc.sh
+++ b/dev/release/release_rc.sh
@@ -30,18 +30,29 @@ fi
rc=$1
+: ${RELEASE_DEFAULT:=1}
+: ${RELEASE_CLEANUP:=${RELEASE_DEFAULT}}
+: ${RELEASE_PULL:=${RELEASE_DEFAULT}}
+: ${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}}
+: ${RELEASE_SIGN:=${RELEASE_DEFAULT}}
+: ${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}
+
cd "${SOURCE_TOP_DIR}"
-git_origin_url="$(git remote get-url origin)"
-if [ "${git_origin_url}" != "git@github.com:apache/arrow-julia.git" ]; then
- echo "This script must be ran with working copy of apache/arrow-julia."
- echo "The origin's URL: ${git_origin_url}"
- exit 1
+if [ ${RELEASE_PULL} -gt 0 -o ${RELEASE_PUSH_TAG} -gt 0 ]; then
+ git_origin_url="$(git remote get-url origin)"
+ if [ "${git_origin_url}" != "git@github.com:apache/arrow-julia.git" ]; then
+ echo "This script must be ran with working copy of apache/arrow-julia."
+ echo "The origin's URL: ${git_origin_url}"
+ exit 1
+ fi
fi
-echo "Ensure using the latest commit"
-git checkout main
-git pull --rebase --prune
+if [ ${RELEASE_PULL} -gt 0 ]; then
+ echo "Ensure using the latest commit"
+ git checkout main
+ git pull --rebase --prune
+fi
version=$(grep -o '^version = ".*"' "Project.toml" | \
sed -e 's/^version = "//' \
@@ -50,7 +61,9 @@ version=$(grep -o '^version = ".*"' "Project.toml" | \
rc_tag="v${version}-rc${rc}"
echo "Tagging for RC: ${rc_tag}"
git tag -a -m "${version} RC${rc}" "${rc_tag}"
-git push origin "${rc_tag}"
+if [ ${RELEASE_PUSH_TAG} -gt 0 ]; then
+ git push origin "${rc_tag}"
+fi
rc_hash="$(git rev-list --max-count=1 "${rc_tag}")"
@@ -75,8 +88,10 @@ pushd "${dev_dist_dir}/${rc_id}"
echo "Running Rat license checker on ${tar_gz}"
../../run_rat.sh ${tar_gz}
-echo "Signing tar.gz and creating checksums"
-gpg --armor --output ${tar_gz}.asc --detach-sig ${tar_gz}
+if [ ${RELEASE_SIGN} -gt 0 ]; then
+ echo "Signing tar.gz and creating checksums"
+ gpg --armor --output ${tar_gz}.asc --detach-sig ${tar_gz}
+fi
if type shasum >/dev/null 2>&1; then
sha256_generate="shasum -a 256"
@@ -88,14 +103,18 @@ fi
${sha256_generate} ${tar_gz} > ${tar_gz}.sha256
${sha512_generate} ${tar_gz} > ${tar_gz}.sha512
-echo "Uploading to ${rc_url}"
-svn add .
-svn ci -m "Apache Arrow Julia ${version} ${rc}"
+if [ ${RELEASE_UPLOAD} -gt 0 ]; then
+ echo "Uploading to ${rc_url}"
+ svn add .
+ svn ci -m "Apache Arrow Julia ${version} ${rc}"
+fi
popd
-echo "Removing temporary directory"
-rm -rf "${dev_dist_dir}"
+if [ ${RELEASE_CLEANUP} -gt 0 ]; then
+ echo "Removing temporary directory"
+ rm -rf "${dev_dist_dir}"
+fi
echo "Draft email for dev@arrow.apache.org mailing list"
echo ""
diff --git a/dev/release/run_rat.sh b/dev/release/run_rat.sh
index e17c8e8..07c6552 100755
--- a/dev/release/run_rat.sh
+++ b/dev/release/run_rat.sh
@@ -28,6 +28,7 @@ if [ ! -f "${RAT_JAR}" ]; then
curl \
--fail \
--output "${RAT_JAR}" \
+ --show-error \
--silent \
https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar
fi
diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh
new file mode 100755
index 0000000..bf2103c
--- /dev/null
+++ b/dev/release/verify_rc.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_SOURCE_DIR="$(dirname $(dirname ${SOURCE_DIR}))"
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 <version> <rc>"
+ echo " e.g.: $0 2.2.1 1"
+ exit 1
+fi
+
+set -o pipefail
+set -x
+
+VERSION="$1"
+RC="$2"
+
+ARROW_DIST_URL="https://dist.apache.org/repos/dist/dev/arrow"
+ARCHIVE_BASE_NAME="apache-arrow-julia-${VERSION}"
+
+: ${VERIFY_DEFAULT:=1}
+: ${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}}
+: ${VERIFY_FORCE_USE_JULIA_BINARY:=0}
+: ${VERIFY_SIGN:=${VERIFY_DEFAULT}}
+
+download_dist_file() {
+ curl \
+ --fail \
+ --location \
+ --remote-name \
+ --show-error \
+ --silent \
+ "${ARROW_DIST_URL}/$1"
+}
+
+download_rc_file() {
+ local path="apache-arrow-julia-${VERSION}-rc${RC}/$1"
+ if [ ${VERIFY_DOWNLOAD} -gt 0 ]; then
+ download_dist_file "${path}"
+ else
+ cp "${SOURCE_DIR}/dist/${path}" "$1"
+ fi
+}
+
+import_gpg_keys() {
+ if [ ${VERIFY_SIGN} -gt 0 ]; then
+ download_dist_file KEYS
+ gpg --import KEYS
+ fi
+}
+
+if type shasum >/dev/null 2>&1; then
+ sha256_verify="shasum -a 256 -c"
+ sha512_verify="shasum -a 512 -c"
+else
+ sha256_verify="sha256sum -c"
+ sha512_verify="sha512sum -c"
+fi
+
+fetch_archive() {
+ download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz
+ if [ ${VERIFY_SIGN} -gt 0 ]; then
+ download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.asc
+ gpg --verify ${ARCHIVE_BASE_NAME}.tar.gz.asc ${ARCHIVE_BASE_NAME}.tar.gz
+ fi
+ download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.sha256
+ ${sha256_verify} ${ARCHIVE_BASE_NAME}.tar.gz.sha256
+ download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.sha512
+ ${sha512_verify} ${ARCHIVE_BASE_NAME}.tar.gz.sha512
+}
+
+setup_tmpdir() {
+ cleanup() {
+ if [ "${VERIFY_SUCCESS}" = "yes" ]; then
+ rm -rf "${VERIFY_TMPDIR}"
+ else
+ echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details."
+ fi
+ }
+
+ if [ -z "${VERIFY_TMPDIR:-}" ]; then
+ VERIFY_TMPDIR=$(mktemp -d -t "$1.XXXXX")
+ trap cleanup EXIT
+ else
+ mkdir -p "${VERIFY_TMPDIR}"
+ fi
+}
+
+latest_julia_version() {
+ curl \
+ --fail \
+ --location \
+ --show-error \
+ --silent \
+ https://api.github.com/repos/JuliaLang/julia/releases | \
+ grep -o '"tag_name": "v.*"' | \
+ head -n 1 | \
+ sed -e 's/^"tag_name": "v//g' \
+ -e 's/"$//g'
+}
+
+ensure_julia() {
+ if [ ${VERIFY_FORCE_USE_JULIA_BINARY} -le 0 ]; then
+ if julia --version; then
+ return
+ fi
+ fi
+
+ local julia_binary_url=https://julialang-s3.julialang.org/bin
+ local julia_version=$(latest_julia_version)
+ local julia_version_series=${julia_version%.*}
+ case "$(uname)" in
+ Darwin)
+ julia_binary_url+="/mac"
+ case "$(arch)" in
+ # TODO
+ # aarch64)
+ # julia_binary_url+="/aarch64"
+ # julia_binary_url+="/${julia_version_series}"
+ # julia_binary_url+="/julia-${julia_version}-macaarch64.dmg"
+ # ;;
+ i386)
+ julia_binary_url+="/x64"
+ julia_binary_url+="/${julia_version_series}"
+ julia_binary_url+="/julia-${julia_version}-mac64.tar.gz"
+ ;;
+ *)
+ echo "You must install Julia manually on $(uname) $(arch)"
+ ;;
+ esac
+ ;;
+ Linux)
+ julia_binary_url+="/linux"
+ case "$(arch)" in
+ aarch64)
+ julia_binary_url+="/aarch64"
+ ;;
+ x86_64)
+ julia_binary_url+="/x64"
+ ;;
+ *)
+ echo "You must install Julia manually on $(uname) $(arch)"
+ ;;
+ esac
+ julia_binary_url+="/${julia_version_series}"
+ julia_binary_url+="/julia-${julia_version}-linux-$(arch).tar.gz"
+ ;;
+ *)
+ echo "You must install Julia manually on $(uname)"
+ exit 1
+ ;;
+ esac
+ julia_binary_tar_gz=$(basename ${julia_binary_url})
+ curl \
+ --fail \
+ --location \
+ --output ${julia_binary_tar_gz} \
+ --show-error \
+ --silent \
+ ${julia_binary_url}
+ tar xf ${julia_binary_tar_gz}
+ julia_path=$(echo julia-*/bin/julia)
+ PATH="$(pwd)/$(dirname ${julia_path}):${PATH}"
+ export JULIA_DEPOT_PATH="$(pwd)/.julia"
+}
+
+test_source_distribution() {
+ julia --project -e 'import Pkg; Pkg.build(); Pkg.test()'
+}
+
+VERIFY_SUCCESS=no
+
+setup_tmpdir "arrow-julia-${VERSION}-${RC}"
+echo "Working in sandbox ${VERIFY_TMPDIR}"
+cd "${VERIFY_TMPDIR}"
+
+import_gpg_keys
+fetch_archive
+tar xf ${ARCHIVE_BASE_NAME}.tar.gz
+ensure_julia
+pushd ${ARCHIVE_BASE_NAME}
+test_source_distribution
+popd
+
+VERIFY_SUCCESS=yes
+echo "RC looks good!"