You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/12/24 03:16:34 UTC
[arrow-adbc] branch main updated: chore: add support for releasing Linux packages (#261)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new fcf871f chore: add support for releasing Linux packages (#261)
fcf871f is described below
commit fcf871fd91b61cac1ffa02839b2aa0d525217bfa
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sat Dec 24 12:16:29 2022 +0900
chore: add support for releasing Linux packages (#261)
Fixes #259.
---
.github/workflows/packaging.yml | 9 +++
ci/linux-packages/Rakefile | 24 +------
dev/release/05-linux-upload.sh | 73 ++++++++++++++++++++++
.../{05-binary-verify.sh => 06-binary-verify.sh} | 0
.../{05-binary-verify.sh => post-05-linux.sh} | 53 ++++++++--------
...rtifacts.sh => post-06-remove-old-artifacts.sh} | 0
...6-bump-versions.sh => post-07-bump-versions.sh} | 0
dev/release/verify-release-candidate.sh | 41 ++++++++++++
docs/source/development/releasing.rst | 32 +++++++---
9 files changed, 175 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index b563faf..8fec095 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -163,6 +163,12 @@ jobs:
distribution=$(echo ${{ matrix.target }} | cut -d- -f1)
echo "DISTRIBUTION=${distribution}" >> $GITHUB_ENV
+ if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+ VERSION=${GITHUB_REF_NAME#apache-arrow-adbc-}
+ VERSION=${VERSION%-rc*}
+ echo "VERSION=${VERSION}" >> $GITHUB_ENV
+ fi
+
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
@@ -585,6 +591,9 @@ jobs:
-name '*.pom' -or \
-name '*.whl' -or \
-name 'adbc_*.tar.gz' \
+ -name 'almalinux-*.tar.gz' \
+ -name 'debian-*.tar.gz' \
+ -name 'ubuntu-*.tar.gz' \
')' \
-exec mv '{}' upload-staging \;
diff --git a/ci/linux-packages/Rakefile b/ci/linux-packages/Rakefile
index 5d1ba98..fd36270 100644
--- a/ci/linux-packages/Rakefile
+++ b/ci/linux-packages/Rakefile
@@ -71,14 +71,7 @@ class ADBCPackageTask < PackageTask
private
def define_archive_task
file @archive_name do
- case @version
- when /\A\d+\.\d+\.\d+-rc\d+\z/
- download_rc_archive
- when /\A\d+\.\d+\.\d+\z/
- download_released_archive
- else
- build_archive
- end
+ build_archive
end
if deb_archive_name != @archive_name
@@ -94,21 +87,6 @@ class ADBCPackageTask < PackageTask
end
end
- def download_rc_archive
- base_url = "https://dist.apache.org/repos/dist/dev/arrow"
- archive_name_no_rc = @archive_name.gsub(/-rc\d+(\.tar\.gz)\z/, "\\1")
- url = "#{base_url}/#{@package}-#{@version}/#{archive_name_no_rc}"
- download(url, @archive_name)
- end
-
- def download_released_archive
- base_url = "https://www.apache.org/dyn/closer.lua/arrow"
- archive_name_no_rc = @archive_name.gsub(/-rc\d+(\.tar\.gz)\z/, "\\1")
- url = "#{base_url}/#{@package}-#{@version}/#{archive_name_no_rc}"
- url += "?action=download"
- download(url, @archive_name)
- end
-
def build_archive
cd(top_source_dir) do
sh("git",
diff --git a/dev/release/05-linux-upload.sh b/dev/release/05-linux-upload.sh
new file mode 100755
index 0000000..be75cfa
--- /dev/null
+++ b/dev/release/05-linux-upload.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -u
+set -o pipefail
+
+main() {
+ local -r source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ local -r source_top_dir="$(cd "${source_dir}/../../" && pwd)"
+
+ if [ $# -ne 3 ]; then
+ echo "Usage: $0 <arrow-dir> <version> <rc-number>"
+ echo "Usage: $0 ../arrow 1.0.0 0"
+ exit
+ fi
+
+ local -r arrow_dir="$(cd "$1" && pwd)"
+ local -r version="$2"
+ local -r rc_number="$3"
+ local -r tag="apache-arrow-adbc-${version}-rc${rc_number}"
+
+ : ${REPOSITORY:="apache/arrow-adbc"}
+
+ export ARROW_ARTIFACTS_DIR="$(pwd)/packages/${tag}/linux"
+ rm -rf "${ARROW_ARTIFACTS_DIR}"
+ mkdir -p "${ARROW_ARTIFACTS_DIR}"
+ gh release download \
+ --dir "${ARROW_ARTIFACTS_DIR}" \
+ --pattern "almalinux-*.tar.gz" \
+ --pattern "debian-*.tar.gz" \
+ --pattern "ubuntu-*.tar.gz" \
+ --repo "${REPOSITORY}" \
+ "${tag}"
+
+ pushd "${ARROW_ARTIFACTS_DIR}"
+ local base_dir=
+ for tar_gz in *.tar.gz; do
+ mkdir -p tmp
+ tar xf ${tar_gz} -C tmp
+ base_dir=${tar_gz%.tar.gz}
+ mkdir -p ${base_dir}
+ find tmp -type f -print0 | xargs -0 mv -t ${base_dir}
+ rm -rf tmp
+ rm -f ${tar_gz}
+ done
+ popd
+
+ export DEB_PACKAGE_NAME=apache-arrow-adbc
+ export UPLOAD_DEFAULT=0
+ export UPLOAD_ALMALINUX=${UPLOAD_ALMALINUX:-1}
+ export UPLOAD_DEBIAN=${UPLOAD_DEBIAN:-1}
+ export UPLOAD_UBUNTU=${UPLOAD_UBUNTU:-1}
+ "${arrow_dir}/dev/release/05-binary-upload.sh" "${version}" "${rc_number}"
+}
+
+main "$@"
diff --git a/dev/release/05-binary-verify.sh b/dev/release/06-binary-verify.sh
similarity index 100%
copy from dev/release/05-binary-verify.sh
copy to dev/release/06-binary-verify.sh
diff --git a/dev/release/05-binary-verify.sh b/dev/release/post-05-linux.sh
similarity index 50%
rename from dev/release/05-binary-verify.sh
rename to dev/release/post-05-linux.sh
index d4a17d8..837527d 100755
--- a/dev/release/05-binary-verify.sh
+++ b/dev/release/post-05-linux.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,41 +17,39 @@
# specific language governing permissions and limitations
# under the License.
-set -euo pipefail
+set -e
+set -u
+set -o pipefail
main() {
local -r source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
local -r source_top_dir="$( cd "${source_dir}/../../" && pwd )"
- local -r version="$1"
- local -r rc_number="$2"
- local -r tag="apache-arrow-adbc-${version}-rc${rc_number}"
+ if [ "$#" -ne 3 ]; then
+ echo "Usage: $0 <arrow-dir> <version> <rc-num>"
+ echo "Usage: $0 ../arrow 1.0.0 0"
+ exit 1
+ fi
+
+ local -r arrow_dir="$(cd "$1" && pwd)"
+ local -r version="$2"
+ local -r rc_number="$3"
: ${REPOSITORY:="apache/arrow-adbc"}
- echo "Starting GitHub Actions workflow on ${REPOSITORY} for ${version} RC${rc_number}"
-
- gh workflow run \
- --repo "${REPOSITORY}" \
- --ref "${tag}" \
- verify.yml \
- --raw-field version="${version}" \
- --raw-field rc="${rc_number}"
-
- local run_id=""
- while [[ -z "${run_id}" ]]
- do
- echo "Waiting for run to start..."
- run_id=$(gh run list \
- --repo "${REPOSITORY}" \
- --workflow=verify.yml \
- --json 'databaseId,event,headBranch,status' \
- --jq ".[] | select(.event == \"workflow_dispatch\" and .headBranch == \"${tag}\" and .status != \"completed\") | .databaseId")
- sleep 1
- done
-
- echo "Started GitHub Actions workflow with ID: ${run_id}"
- echo "You can wait for completion via: gh run watch --repo ${REPOSITORY} ${run_id}"
+ header "Deploying APT/Yum repositories ${version}"
+
+ export DEPLOY_DEFAULT=0
+ export DEPLOY_ALMALINUX=${DEPLOY_ALMALINUX:-1}
+ export DEPLOY_DEBIAN=${DEPLOY_DEBIAN:-1}
+ export DEPLOY_UBUNTU=${DEPLOY_UBUNTU:-1}
+ "${arrow_dir}/dev/release/post-02-binary.sh" "${version}" "${rc_number}"
+}
+
+header() {
+ echo "============================================================"
+ echo "${1}"
+ echo "============================================================"
}
main "$@"
diff --git a/dev/release/post-05-remove-old-artifacts.sh b/dev/release/post-06-remove-old-artifacts.sh
similarity index 100%
rename from dev/release/post-05-remove-old-artifacts.sh
rename to dev/release/post-06-remove-old-artifacts.sh
diff --git a/dev/release/post-06-bump-versions.sh b/dev/release/post-07-bump-versions.sh
similarity index 100%
rename from dev/release/post-06-bump-versions.sh
rename to dev/release/post-07-bump-versions.sh
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 96e858e..9031a46 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -167,10 +167,51 @@ test_binary() {
test_apt() {
show_header "Testing APT packages"
+
+ local verify_type=rc
+ if [ "${TEST_STAGING:-0}" -gt 0 ]; then
+ verify_type=staging-${verify_type}
+ fi
+ for target in "debian:bullseye" \
+ "debian:bookworm" \
+ "ubuntu:jammy"; do \
+ show_info "Verifying ${target}..."
+ if ! docker run \
+ --rm \
+ --security-opt="seccomp=unconfined" \
+ --volume "${ADBC_DIR}":/host:delegated \
+ "${target}" \
+ /host/dev/release/verify-apt.sh \
+ "${VERSION}" \
+ "${verify_type}"; then
+ echo "Failed to verify the APT repository for ${target}"
+ exit 1
+ fi
+ done
}
test_yum() {
show_header "Testing Yum packages"
+
+ local verify_type=rc
+ if [ "${TEST_STAGING:-0}" -gt 0 ]; then
+ verify_type=staging-${verify_type}
+ fi
+ for target in "almalinux:9" \
+ "almalinux:8"; do
+ show_info "Verifying ${target}..."
+ if ! docker run \
+ --rm \
+ --security-opt="seccomp=unconfined" \
+ --volume "${ADBC_DIR}":/host:delegated \
+ "${target}" \
+ /host/dev/release/verify-yum.sh \
+ "${VERSION}" \
+ "${verify_type}"; then
+ echo "Failed to verify the Yum repository for ${target}"
+ exit 1
+ fi
+ done
}
setup_tempdir() {
diff --git a/docs/source/development/releasing.rst b/docs/source/development/releasing.rst
index 58ee2a1..a484ed5 100644
--- a/docs/source/development/releasing.rst
+++ b/docs/source/development/releasing.rst
@@ -153,8 +153,14 @@ Build source and binaries and submit them
# https://repository.apache.org/#stagingRepositories
dev/release/04-java-upload.sh <version> <rc-number>
+ # Sign and upload the deb/rpm packages and APT/Yum repositories
+ #
+ # This reuses release scripts in apache/arrow. So you need to
+ # specify cloned apache/arrow directory.
+ dev/release/05-linux-upload.sh <arrow-dir> <version> <rc-number>
+
# Start verifications for binaries and wheels
- dev/release/05-binary-verify.sh <version> <rc-number>
+ dev/release/06-binary-verify.sh <version> <rc-number>
Verify the Release
------------------
@@ -199,7 +205,7 @@ Be sure to go through on the following checklist:
.. code-block:: Bash
# dev/release/post-01-upload.sh 0.1.0 0
- dev/release/post-01-upload.sh <version> <rc>
+ dev/release/post-01-upload.sh <version> <rc-number>
git push --tag apache apache-arrow-adbc-<version>
.. dropdown:: Create the final GitHub release
@@ -211,7 +217,7 @@ Be sure to go through on the following checklist:
.. code-block:: Bash
# dev/release/post-02-binary.sh 0.1.0 0
- dev/release/post-02-binary.sh <version> <rc number>
+ dev/release/post-02-binary.sh <version> <rc-number>
.. dropdown:: Update website
:class-title: sd-fs-5
@@ -246,7 +252,19 @@ Be sure to go through on the following checklist:
.. code-block:: Bash
# dev/release/post-04-go.sh 10.0.0
- dev/release/post-04-go.sh X.Y.Z
+ dev/release/post-04-go.sh <version>
+
+.. dropdown:: Deploy APT/Yum repositories
+ :class-title: sd-fs-5
+ :class-container: sd-shadow-md
+
+ .. code-block:: Bash
+
+ # This reuses release scripts in apache/arrow. So you need to
+ # specify cloned apache/arrow directory.
+ #
+ # dev/release/post-05-linux.sh ../arrow 10.0.0 0
+ dev/release/post-05-linux.sh <arrow-dir> <version> <rc-number>
.. dropdown:: Announce the new release
:class-title: sd-fs-5
@@ -270,7 +288,7 @@ Be sure to go through on the following checklist:
.. code-block:: Bash
- dev/release/post-05-remove-old-artifacts.sh
+ dev/release/post-06-remove-old-artifacts.sh
.. dropdown:: Bump versions
:class-title: sd-fs-5
@@ -278,7 +296,7 @@ Be sure to go through on the following checklist:
.. code-block:: Bash
- # dev/release/post-06-bump-versions.sh 0.1.0 0.2.0
- dev/release/post-06-bump-versions.sh <version> <next_version>
+ # dev/release/post-07-bump-versions.sh 0.1.0 0.2.0
+ dev/release/post-07-bump-versions.sh <version> <next_version>
.. _nightly-website.yml: https://github.com/apache/arrow-adbc/actions/workflows/nightly-website.yml