You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/02/25 21:22:51 UTC
[arrow] branch master updated: ARROW-4655: [Packaging] Parallelize
binary upload
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c6791d9 ARROW-4655: [Packaging] Parallelize binary upload
c6791d9 is described below
commit c6791d9d031a275f15c655fd56556c851b8506b0
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Mon Feb 25 15:22:41 2019 -0600
ARROW-4655: [Packaging] Parallelize binary upload
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #3730 from kou/release-binary-paralell and squashes the following commits:
82ec7637 <Kouhei Sutou> Too parallel
3768cf44 <Kouhei Sutou> Too parallel
86e781b3 <Kouhei Sutou> More parallel
50f7187e <Kouhei Sutou> Resolve conflicts
57af6272 <Kouhei Sutou> Parallelize binary upload
---
dev/release/03-binary.sh | 160 +++++++++++++++++++++++++-----------------
dev/release/binary/Dockerfile | 2 +
2 files changed, 97 insertions(+), 65 deletions(-)
diff --git a/dev/release/03-binary.sh b/dev/release/03-binary.sh
index 976b317..01f5a10 100755
--- a/dev/release/03-binary.sh
+++ b/dev/release/03-binary.sh
@@ -19,6 +19,7 @@
# under the License.
#
set -e
+set -u
set -o pipefail
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -34,7 +35,6 @@ gpg_key_id=$3
artifact_dir=$4
docker_image_name=apache-arrow/release-binary
-docker_gpg_ssh_port=10022
gpg_agent_extra_socket="$(gpgconf --list-dirs agent-extra-socket)"
if [ $(uname) = "Darwin" ]; then
docker_uid=10000
@@ -80,27 +80,26 @@ docker_run() {
}
docker_gpg_ssh() {
+ local ssh_port=$1
+ shift
ssh \
-o StrictHostKeyChecking=no \
-i "${docker_ssh_key}" \
- -p ${docker_gpg_ssh_port} \
+ -p ${ssh_port} \
-R "/home/arrow/.gnupg/S.gpg-agent:${gpg_agent_extra_socket}" \
arrow@127.0.0.1 \
"$@"
}
docker_run_gpg_ready() {
- local docker_container_id_file=/tmp/arrow-binary-gpg-container-id
- if [ -f ${docker_container_id_file} ]; then
- docker kill $(cat ${docker_container_id_file}) || :
- rm -rf ${docker_container_id_file}
- fi
+ local container_id_dir=$(mktemp -d -t "arrow-binary-gpg-container.XXXXX")
+ local container_id_file=${container_id_dir}/id
docker \
run \
--rm \
--detach \
- --cidfile ${docker_container_id_file} \
- --publish ${docker_gpg_ssh_port}:22 \
+ --cidfile ${container_id_file} \
+ --publish-all \
--volume "$PWD":/host \
${docker_image_name} \
bash -c "
@@ -110,11 +109,16 @@ if [ \$(id -u) -ne ${docker_uid} ]; then
fi
/usr/sbin/sshd -D
"
- sleep 1 # Wait for sshd available
- gpg --export ${gpg_key_id} | docker_gpg_ssh gpg --import
- docker_gpg_ssh "cd /host && $@"
- docker kill $(cat ${docker_container_id_file})
- rm -f ${docker_container_id_file}
+ local container_id=$(cat ${container_id_file})
+ local ssh_port=$(docker port ${container_id} | grep -E -o '[0-9]+$')
+ # Wait for sshd available
+ while ! docker_gpg_ssh ${ssh_port} : > /dev/null 2>&1; do
+ sleep 0.1
+ done
+ gpg --export ${gpg_key_id} | docker_gpg_ssh ${ssh_port} gpg --import
+ docker_gpg_ssh ${ssh_port} "cd /host && $@"
+ docker kill ${container_id}
+ rm -rf ${container_id_dir}
}
jq() {
@@ -183,7 +187,7 @@ download_files() {
--fail \
--location \
--output ${file} \
- https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file}
+ https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file} &
done
}
@@ -238,10 +242,11 @@ sign_and_upload_file() {
for suffix in asc sha256 sha512; do
pushd $(dirname ${local_path})
local local_path_base=$(basename ${local_path})
- local output=tmp.${suffix}
+ local output_dir=$(mktemp -d -t "arrow-binary-sign.XXXXX")
+ local output=${output_dir}/tmp.${suffix}
case $suffix in
asc)
- docker_run_gpg_ready gpg \
+ gpg \
--local-user ${gpg_key_id} \
--detach-sig \
--output ${output} \
@@ -254,34 +259,42 @@ sign_and_upload_file() {
;;
esac
upload_file ${version} ${rc} ${target} ${output} ${upload_path}.${suffix}
- rm -f ${output}
+ rm -rf ${output_dir}
popd
done
}
-upload_deb() {
+upload_deb_file() {
local version=$1
local rc=$2
local distribution=$3
local code_name=$4
+ local base_path=$5
+
+ case ${base_path} in
+ *.dsc|*.changes)
+ docker_run_gpg_ready debsign -k${gpg_key_id} --re-sign ${base_path}
+ ;;
+ *.asc|*.sha256|*.sha512)
+ return
+ ;;
+ esac
+ sign_and_upload_file \
+ ${version} \
+ ${rc} \
+ ${distribution} \
+ ${base_path} \
+ pool/${code_name}/main/a/apache-arrow/${base_path}
+}
- ensure_version ${version} ${rc} ${distribution}
+upload_deb() {
+ local version=$1
+ local rc=$2
+ local distribution=$3
+ local code_name=$4
for base_path in *; do
- case ${base_path} in
- *.dsc|*.changes)
- docker_run_gpg_ready debsign -k${gpg_key_id} --re-sign ${base_path}
- ;;
- *.asc|*.sha256|*.sha512)
- continue
- ;;
- esac
- sign_and_upload_file \
- ${version} \
- ${rc} \
- ${distribution} \
- ${base_path} \
- pool/${code_name}/main/a/apache-arrow/${base_path}
+ upload_deb_file ${version} ${rc} ${distribution} ${code_name} ${base_path} &
done
}
@@ -296,6 +309,7 @@ upload_apt() {
pushd ${tmp_dir}
download_files ${version} ${rc} ${distribution}
+ wait
pushd ${distribution}-rc
@@ -311,7 +325,7 @@ upload_apt() {
${rc} \
${distribution} \
${keyring_name} \
- ${keyring_name}
+ ${keyring_name} &
for pool_code_name in pool/*; do
local code_name=$(basename ${pool_code_name})
@@ -355,47 +369,59 @@ upload_apt() {
${rc} \
${distribution} \
${path} \
- ${path}
+ ${path} &
done
+ wait
done
-
popd
popd
rm -rf ${tmp_dir}
}
-upload_rpm() {
+upload_rpm_file() {
local version=$1
local rc=$2
local distribution=$3
local distribution_version=$4
+ local rpm_path=$5
- local version_name=${version}-rc${rc}
+ local upload_path=${distribution_version}
+
+ case ${rpm_path} in
+ *.src.rpm)
+ upload_path=${upload_path}/Source/SPackages
+ ;;
+ *)
+ upload_path=${upload_path}/x86_64/Packages
+ ;;
+ esac
+ upload_path=${upload_path}/${rpm_path}
+ docker_run_gpg_ready rpm \
+ -D "_gpg_name\\ ${gpg_key_id}" \
+ --addsign \
+ ${rpm_path}
+ sign_and_upload_file \
+ ${version} \
+ ${rc} \
+ ${distribution} \
+ ${rpm_path} \
+ ${upload_path}
+}
- ensure_version ${version} ${rc} ${distribution}
+upload_rpm() {
+ local version=$1
+ local rc=$2
+ local distribution=$3
+ local distribution_version=$4
for rpm_path in *.rpm; do
- local upload_path=${distribution_version}
- case ${base_path} in
- *.src.rpm)
- upload_path=${upload_path}/Source/SPackages
- ;;
- *)
- upload_path=${upload_path}/x86_64/Packages
- ;;
- esac
- upload_path=${upload_path}/${rpm_path}
- docker_run_gpg_ready rpm \
- -D "_gpg_name\\ ${gpg_key_id}" \
- --addsign \
- ${rpm_path}
- sign_and_upload_file \
+ upload_rpm_file \
${version} \
${rc} \
${distribution} \
- ${rpm_path} \
- ${upload_path}
+ ${distribution_version} \
+ ${rpm_path} &
done
}
@@ -412,6 +438,7 @@ upload_yum() {
pushd ${tmp_dir}
download_files ${version} ${rc} ${distribution}
+ wait
pushd ${distribution}-rc
local keyring_name=RPM-GPG-KEY-apache-arrow
@@ -421,7 +448,7 @@ upload_yum() {
${rc} \
${distribution} \
${keyring_name} \
- ${keyring_name}
+ ${keyring_name} &
for version_dir in $(find . -mindepth 1 -maxdepth 1 -type d); do
for arch_dir in ${version_dir}/*; do
mkdir -p ${arch_dir}/repodata/
@@ -432,10 +459,11 @@ upload_yum() {
${rc} \
${distribution} \
${repo_path} \
- ${repo_path}
+ ${repo_path} &
done
done
done
+ wait
popd
popd
@@ -447,8 +475,6 @@ upload_python() {
local rc=$2
local target=python
- ensure_version ${version} ${rc} ${target}
-
for base_path in *; do
case ${base_path} in
*.asc|*.sha256|*.sha512)
@@ -460,7 +486,7 @@ upload_python() {
${rc} \
${target} \
${base_path} \
- ${version}-rc${rc}/${base_path}
+ ${version}-rc${rc}/${base_path} &
done
}
@@ -504,18 +530,22 @@ for dir in *; do
if [ ${is_deb} = "yes" ]; then
pushd ${dir}
- upload_deb ${version} ${rc} ${distribution} ${code_name}
+ ensure_version ${version} ${rc} ${distribution}
+ upload_deb ${version} ${rc} ${distribution} ${code_name} &
popd
elif [ ${is_rpm} = "yes" ]; then
pushd ${dir}
- upload_rpm ${version} ${rc} ${distribution} ${distribution_version}
+ ensure_version ${version} ${rc} ${distribution}
+ upload_rpm ${version} ${rc} ${distribution} ${distribution_version} &
popd
elif [ ${is_python} = "yes" ]; then
pushd ${dir}
- upload_python ${version} ${rc}
+ ensure_version ${version} ${rc} python
+ upload_python ${version} ${rc} &
popd
fi
done
+wait
popd
if [ ${have_debian} = "yes" ]; then
diff --git a/dev/release/binary/Dockerfile b/dev/release/binary/Dockerfile
index 0470206..372f084 100644
--- a/dev/release/binary/Dockerfile
+++ b/dev/release/binary/Dockerfile
@@ -53,3 +53,5 @@ COPY id_rsa.pub /home/arrow/.ssh/authorized_keys
RUN \
chown -R arrow: /home/arrow/.ssh && \
chmod -R og-rwx /home/arrow/.ssh
+
+EXPOSE 22