You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/12/18 20:22:44 UTC
[arrow] branch master updated: GH-14992: [Packaging] Make dev/release/binary-task.rb reusable from other project (#14994)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f82501e763 GH-14992: [Packaging] Make dev/release/binary-task.rb reusable from other project (#14994)
f82501e763 is described below
commit f82501e763ff48af610077f9525ae83cc3ab2e95
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Mon Dec 19 05:22:34 2022 +0900
GH-14992: [Packaging] Make dev/release/binary-task.rb reusable from other project (#14994)
I'll use dev/release/binary-task.rb from apache/arrow-adbc.
* Closes: #14992
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
dev/release/binary-task.rb | 192 +++++++++++++++++++++++++++++++
dev/release/verify-apt.sh | 8 +-
dev/release/verify-yum.sh | 8 +-
dev/tasks/linux-packages/Rakefile | 184 +----------------------------
dev/tasks/linux-packages/helper.rb | 14 +++
dev/tasks/linux-packages/package-task.rb | 14 ---
6 files changed, 219 insertions(+), 201 deletions(-)
diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb
index 6597b93bdb..9b9bb6ed86 100644
--- a/dev/release/binary-task.rb
+++ b/dev/release/binary-task.rb
@@ -2005,3 +2005,195 @@ Success! The release binaries are available here:
end
end
end
+
+class LocalBinaryTask < BinaryTask
+ def initialize(packages, top_source_directory)
+ @packages = packages
+ @top_source_directory = top_source_directory
+ super()
+ end
+
+ def define
+ define_apt_test_task
+ define_yum_test_task
+ end
+
+ private
+ def resolve_docker_image(target)
+ case target
+ when /-(?:arm64|aarch64)\z/
+ target = Regexp.last_match.pre_match
+ platform = "linux/arm64"
+ else
+ platform = "linux/amd64"
+ end
+
+ case target
+ when /\Acentos-(\d+)-stream\z/
+ centos_stream_version = $1
+ image = "quay.io/centos/centos:stream#{centos_stream_version}"
+ else
+ case platform
+ when "linux/arm64"
+ image = "arm64v8/"
+ else
+ image = ""
+ end
+ target = target.gsub(/\Aamazon-linux/, "amazonlinux")
+ image << target.gsub(/-/, ":")
+ end
+
+ [platform, image]
+ end
+
+ def verify_apt_sh
+ "/host/dev/release/verify-apt.sh"
+ end
+
+ def verify_yum_sh
+ "/host/dev/release/verify-yum.sh"
+ end
+
+ def verify(target)
+ verify_command_line = [
+ "docker",
+ "run",
+ "--log-driver", "none",
+ "--rm",
+ "--security-opt", "seccomp=unconfined",
+ "--volume", "#{@top_source_directory}:/host:delegated",
+ ]
+ if $stdin.tty?
+ verify_command_line << "--interactive"
+ verify_command_line << "--tty"
+ else
+ verify_command_line.concat(["--attach", "STDOUT"])
+ verify_command_line.concat(["--attach", "STDERR"])
+ end
+ platform, docker_image = resolve_docker_image(target)
+ docker_info = JSON.parse(`docker info --format '{{json .}}'`)
+ case [platform, docker_info["Architecture"]]
+ when ["linux/amd64", "x86_64"],
+ ["linux/arm64", "aarch64"]
+ # Do nothing
+ else
+ verify_command_line.concat(["--platform", platform])
+ end
+ verify_command_line << docker_image
+ case target
+ when /\Adebian-/, /\Aubuntu-/
+ verify_command_line << verify_apt_sh
+ else
+ verify_command_line << verify_yum_sh
+ end
+ verify_command_line << version
+ verify_command_line << "local"
+ sh(*verify_command_line)
+ end
+
+ def apt_test_targets
+ targets = (ENV["APT_TARGETS"] || "").split(",")
+ targets = apt_test_targets_default if targets.empty?
+ targets
+ end
+
+ def apt_test_targets_default
+ # Disable arm64 targets by default for now
+ # because they require some setups on host.
+ [
+ "debian-buster",
+ # "debian-buster-arm64",
+ "debian-bullseye",
+ # "debian-bullseye-arm64",
+ "debian-bookworm",
+ # "debian-bookworm-arm64",
+ "ubuntu-bionic",
+ # "ubuntu-bionic-arm64",
+ "ubuntu-focal",
+ # "ubuntu-focal-arm64",
+ "ubuntu-impish",
+ # "ubuntu-impish-arm64",
+ ]
+ end
+
+ def define_apt_test_task
+ namespace :apt do
+ desc "Test deb packages"
+ task :test do
+ repositories_dir = "apt/repositories"
+ unless @packages.empty?
+ rm_rf(repositories_dir)
+ @packages.each do |package|
+ package_repositories = "#{package}/apt/repositories"
+ next unless File.exist?(package_repositories)
+ sh("rsync", "-av", "#{package_repositories}/", repositories_dir)
+ end
+ end
+ Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir|
+ universe_dir = "#{code_name_dir}/universe"
+ next unless File.exist?(universe_dir)
+ mv(universe_dir, "#{code_name_dir}/main")
+ end
+ base_dir = "nonexistent"
+ merged_dir = "apt/merged"
+ apt_update(base_dir, repositories_dir, merged_dir)
+ Dir.glob("#{merged_dir}/*/dists/*") do |dists_code_name_dir|
+ prefix = dists_code_name_dir.split("/")[-3..-1].join("/")
+ mv(Dir.glob("#{dists_code_name_dir}/*Release*"),
+ "#{repositories_dir}/#{prefix}")
+ end
+ apt_test_targets.each do |target|
+ verify(target)
+ end
+ end
+ end
+ end
+
+ def yum_test_targets
+ targets = (ENV["YUM_TARGETS"] || "").split(",")
+ targets = yum_test_targets_default if targets.empty?
+ targets
+ end
+
+ def yum_test_targets_default
+ # Disable aarch64 targets by default for now
+ # because they require some setups on host.
+ [
+ "almalinux-9",
+ # "almalinux-9-aarch64",
+ "almalinux-8",
+ # "almalinux-8-aarch64",
+ "amazon-linux-2",
+ # "amazon-linux-2-aarch64",
+ "centos-9-stream",
+ # "centos-9-stream-aarch64",
+ "centos-8-stream",
+ # "centos-8-stream-aarch64",
+ "centos-7",
+ # "centos-7-aarch64",
+ ]
+ end
+
+ def define_yum_test_task
+ namespace :yum do
+ desc "Test RPM packages"
+ task :test do
+ repositories_dir = "yum/repositories"
+ unless @packages.empty?
+ rm_rf(repositories_dir)
+ @packages.each do |package|
+ package_repositories = "#{package}/yum/repositories"
+ next unless File.exist?(package_repositories)
+ sh("rsync", "-av", "#{package_repositories}/", repositories_dir)
+ end
+ end
+ rpm_sign(repositories_dir)
+ base_dir = "nonexistent"
+ yum_update(base_dir, repositories_dir)
+ yum_test_targets.each do |target|
+ verify(target)
+ end
+ end
+ end
+ end
+end
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
index 15e31bf4b4..7503c0df7b 100755
--- a/dev/release/verify-apt.sh
+++ b/dev/release/verify-apt.sh
@@ -36,7 +36,9 @@ fi
VERSION="$1"
TYPE="$2"
-local_prefix="/arrow/dev/tasks/linux-packages"
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_SOURCE_DIR="${SOURCE_DIR}/../.."
+local_prefix="${TOP_SOURCE_DIR}/dev/tasks/linux-packages"
echo "::group::Prepare repository"
@@ -148,7 +150,7 @@ required_packages+=(pkg-config)
required_packages+=(${workaround_missing_packages[@]})
${APT_INSTALL} ${required_packages[@]}
mkdir -p build
-cp -a /arrow/cpp/examples/minimal_build build/
+cp -a "${TOP_SOURCE_DIR}/cpp/examples/minimal_build" build/
pushd build/minimal_build
cmake .
make -j$(nproc)
@@ -166,7 +168,7 @@ ${APT_INSTALL} libarrow-glib-dev=${package_version}
${APT_INSTALL} libarrow-glib-doc=${package_version}
${APT_INSTALL} valac
-cp -a /arrow/c_glib/example/vala build/
+cp -a "${TOP_SOURCE_DIR}/c_glib/example/vala" build/
pushd build/vala
valac --pkg arrow-glib --pkg posix build.vala
./build
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index 521c7a41d8..3a1c0ec6ad 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -36,7 +36,9 @@ fi
VERSION="$1"
TYPE="$2"
-local_prefix="/arrow/dev/tasks/linux-packages"
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_SOURCE_DIR="${SOURCE_DIR}/../.."
+local_prefix="${TOP_SOURCE_DIR}/dev/tasks/linux-packages"
artifactory_base_url="https://apache.jfrog.io/artifactory/arrow"
@@ -216,7 +218,7 @@ else
make
fi
mkdir -p build
-cp -a /arrow/cpp/examples/minimal_build build/
+cp -a "${TOP_SOURCE_DIR}/cpp/examples/minimal_build" build/
pushd build/minimal_build
${cmake_command} .
make -j$(nproc)
@@ -234,7 +236,7 @@ if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
${install_command} vala
- cp -a /arrow/c_glib/example/vala build/
+ cp -a "${TOP_SOURCE_DIR}/c_glib/example/vala" build/
pushd build/vala
valac --pkg arrow-glib --pkg posix build.vala
./build
diff --git a/dev/tasks/linux-packages/Rakefile b/dev/tasks/linux-packages/Rakefile
index 28fc37bb2a..9634bd07f5 100644
--- a/dev/tasks/linux-packages/Rakefile
+++ b/dev/tasks/linux-packages/Rakefile
@@ -84,196 +84,18 @@ namespace :docker do
end
-class LocalBinaryTask < BinaryTask
+class ApacheArrowLocalBinaryTask < LocalBinaryTask
include Helper::ApacheArrow
def initialize(packages)
- @packages = packages
- super()
- end
-
- def define
- define_apt_test_task
- define_yum_test_task
+ super(packages, File.expand_path(arrow_source_dir))
end
private
- def latest_commit_time(git_directory)
- cd(git_directory) do
- return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc
- end
- end
-
def version
@version ||= detect_version(detect_release_time)
end
-
- def resolve_docker_image(target)
- case target
- when /-(?:arm64|aarch64)\z/
- target = Regexp.last_match.pre_match
- platform = "linux/arm64"
- else
- platform = "linux/amd64"
- end
-
- case target
- when /\Acentos-(\d+)-stream\z/
- centos_stream_version = $1
- image = "quay.io/centos/centos:stream#{centos_stream_version}"
- else
- case platform
- when "linux/arm64"
- image = "arm64v8/"
- else
- image = ""
- end
- target = target.gsub(/\Aamazon-linux/, "amazonlinux")
- image << target.gsub(/-/, ":")
- end
-
- [platform, image]
- end
-
- def verify(target)
- verify_command_line = [
- "docker",
- "run",
- "--log-driver", "none",
- "--rm",
- "--security-opt", "seccomp=unconfined",
- "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated",
- ]
- if $stdin.tty?
- verify_command_line << "--interactive"
- verify_command_line << "--tty"
- else
- verify_command_line.concat(["--attach", "STDOUT"])
- verify_command_line.concat(["--attach", "STDERR"])
- end
- platform, docker_image = resolve_docker_image(target)
- docker_info = JSON.parse(`docker info --format '{{json .}}'`)
- case [platform, docker_info["Architecture"]]
- when ["linux/amd64", "x86_64"],
- ["linux/arm64", "aarch64"]
- # Do nothing
- else
- verify_command_line.concat(["--platform", platform])
- end
- verify_command_line << docker_image
- case target
- when /\Adebian-/, /\Aubuntu-/
- verify_command_line << "/arrow/dev/release/verify-apt.sh"
- else
- verify_command_line << "/arrow/dev/release/verify-yum.sh"
- end
- verify_command_line << version
- verify_command_line << "local"
- sh(*verify_command_line)
- end
-
- def apt_test_targets
- targets = (ENV["APT_TARGETS"] || "").split(",")
- targets = apt_test_targets_default if targets.empty?
- targets
- end
-
- def apt_test_targets_default
- # Disable arm64 targets by default for now
- # because they require some setups on host.
- [
- "debian-buster",
- # "debian-buster-arm64",
- "debian-bullseye",
- # "debian-bullseye-arm64",
- "debian-bookworm",
- # "debian-bookworm-arm64",
- "ubuntu-bionic",
- # "ubuntu-bionic-arm64",
- "ubuntu-focal",
- # "ubuntu-focal-arm64",
- "ubuntu-impish",
- # "ubuntu-impish-arm64",
- ]
- end
-
- def define_apt_test_task
- namespace :apt do
- desc "Test deb packages"
- task :test do
- repositories_dir = "apt/repositories"
- rm_rf(repositories_dir)
- @packages.each do |package|
- package_repositories = "#{package}/apt/repositories"
- next unless File.exist?(package_repositories)
- sh("rsync", "-av", "#{package_repositories}/", repositories_dir)
- end
- Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir|
- universe_dir = "#{code_name_dir}/universe"
- next unless File.exist?(universe_dir)
- mv(universe_dir, "#{code_name_dir}/main")
- end
- base_dir = "nonexistent"
- merged_dir = "apt/merged"
- apt_update(base_dir, repositories_dir, merged_dir)
- Dir.glob("#{merged_dir}/*/dists/*") do |dists_code_name_dir|
- prefix = dists_code_name_dir.split("/")[-3..-1].join("/")
- mv(Dir.glob("#{dists_code_name_dir}/*Release*"),
- "#{repositories_dir}/#{prefix}")
- end
- apt_test_targets.each do |target|
- verify(target)
- end
- end
- end
- end
-
- def yum_test_targets
- targets = (ENV["YUM_TARGETS"] || "").split(",")
- targets = yum_test_targets_default if targets.empty?
- targets
- end
-
- def yum_test_targets_default
- # Disable aarch64 targets by default for now
- # because they require some setups on host.
- [
- "almalinux-9",
- # "almalinux-9-aarch64",
- "almalinux-8",
- # "almalinux-8-aarch64",
- "amazon-linux-2",
- # "amazon-linux-2-aarch64",
- "centos-9-stream",
- # "centos-9-stream-aarch64",
- "centos-8-stream",
- # "centos-8-stream-aarch64",
- "centos-7",
- # "centos-7-aarch64",
- ]
- end
-
- def define_yum_test_task
- namespace :yum do
- desc "Test RPM packages"
- task :test do
- repositories_dir = "yum/repositories"
- rm_rf(repositories_dir)
- @packages.each do |package|
- package_repositories = "#{package}/yum/repositories"
- next unless File.exist?(package_repositories)
- sh("rsync", "-av", "#{package_repositories}/", repositories_dir)
- end
- rpm_sign(repositories_dir)
- base_dir = "nonexistent"
- yum_update(base_dir, repositories_dir)
- yum_test_targets.each do |target|
- verify(target)
- end
- end
- end
- end
end
-local_binary_task = LocalBinaryTask.new(packages)
+local_binary_task = ApacheArrowLocalBinaryTask.new(packages)
local_binary_task.define
diff --git a/dev/tasks/linux-packages/helper.rb b/dev/tasks/linux-packages/helper.rb
index 30ac3b8982..03ff845a1d 100644
--- a/dev/tasks/linux-packages/helper.rb
+++ b/dev/tasks/linux-packages/helper.rb
@@ -18,6 +18,20 @@
module Helper
module ApacheArrow
private
+ def git_directory?(directory)
+ candidate_paths = [".git", "HEAD"]
+ candidate_paths.any? do |candidate_path|
+ File.exist?(File.join(directory, candidate_path))
+ end
+ end
+
+ def latest_commit_time(git_directory)
+ return nil unless git_directory?(git_directory)
+ cd(git_directory) do
+ return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc
+ end
+ end
+
def detect_release_time
release_time_env = ENV["ARROW_RELEASE_TIME"]
if release_time_env
diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb
index 6b0ba7de13..aba53c0908 100644
--- a/dev/tasks/linux-packages/package-task.rb
+++ b/dev/tasks/linux-packages/package-task.rb
@@ -77,20 +77,6 @@ class PackageTask
ENV["DEBUG"] != "no"
end
- def git_directory?(directory)
- candidate_paths = [".git", "HEAD"]
- candidate_paths.any? do |candidate_path|
- File.exist?(File.join(directory, candidate_path))
- end
- end
-
- def latest_commit_time(git_directory)
- return nil unless git_directory?(git_directory)
- cd(git_directory) do
- return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc
- end
- end
-
def download(url, output_path)
if File.directory?(output_path)
base_name = url.split("/").last