You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/04/03 11:54:53 UTC
[arrow] branch master updated: ARROW-5094: [Packaging] Add APT/Yum
verification scripts
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 1574b77 ARROW-5094: [Packaging] Add APT/Yum verification scripts
1574b77 is described below
commit 1574b776c138f6d0da468077f7f713ec5d0b55a6
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Wed Apr 3 13:54:33 2019 +0200
ARROW-5094: [Packaging] Add APT/Yum verification scripts
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #4098 from kou/packaing-test-apt-yum-repository and squashes the following commits:
4b47bb3cd <Kouhei Sutou> Add APT/Yum verification scripts
---
dev/release/verify-apt.sh | 124 ++++++++++++++++++++++++++++++++
dev/release/verify-release-candidate.sh | 94 +++++++++++++++++-------
dev/release/verify-yum.sh | 89 +++++++++++++++++++++++
dev/run_docker_compose.sh | 10 +--
docker-compose.yml | 37 ++++++++++
5 files changed, 325 insertions(+), 29 deletions(-)
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
new file mode 100755
index 0000000..5c695f9
--- /dev/null
+++ b/dev/release/verify-apt.sh
@@ -0,0 +1,124 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -exu
+
+if [ $# -lt 2 ]; then
+ echo "Usage: $0 X.Y.Z IS_RC"
+ echo " $0 X.Y.Z IS_RC BINTRAY_REPOSITORY"
+ echo " e.g.: $0 0.13.0 yes # Verify 0.13.0 RC"
+ echo " e.g.: $0 0.13.0 no # Verify 0.13.0"
+ echo " e.g.: $0 0.13.0 yes kou/arrow # Verify 0.13.0 RC at https://bintray.com/kou/arrow"
+ exit 1
+fi
+
+VERSION="$1"
+IS_RC="$2"
+BINTRAY_REPOSITORY="${3:-apache/arrow}"
+
+deb_version="${VERSION}-1"
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt update
+apt install -y -V \
+ apt-transport-https \
+ curl \
+ gnupg \
+ lsb-release
+
+code_name="$(lsb_release --codename --short)"
+distribution="$(lsb_release --id --short | tr 'A-Z' 'a-z')"
+bintray_base_url="https://dl.bintray.com/${BINTRAY_REPOSITORY}/${distribution}"
+if [ "${IS_RC}" = "yes" ]; then
+ bintray_base_url="${bintray_base_url}-rc"
+fi
+
+have_signed_by=yes
+have_python=yes
+have_gandiva=yes
+need_llvm_apt=no
+case "${distribution}-${code_name}" in
+ debian-*)
+ sed \
+ -i"" \
+ -e "s/ main$/ main contrib non-free/g" \
+ /etc/apt/sources.list
+ cat <<APT_LINE > /etc/apt/sources.list.d/backports.list
+deb http://deb.debian.org/debian ${code_name}-backports main
+APT_LINE
+ need_llvm_apt=yes
+ ;;
+ ubuntu-xenial)
+ have_signed_by=no
+ need_llvm_apt=yes
+ ;;
+ ubuntu-trusty)
+ have_signed_by=no
+ have_python=no
+ have_gandiva=no
+ ;;
+esac
+
+if [ "${have_signed_by}" = "yes" ]; then
+ keyring_path="/usr/share/keyrings/apache-arrow-keyring.gpg"
+ curl \
+ --output "${keyring_path}" \
+ "${bintray_base_url}/apache-arrow-keyring.gpg"
+ cat <<APT_LINE > /etc/apt/sources.list.d/apache-arrow.list
+deb [arch=amd64 signed-by=${keyring_path}] ${bintray_base_url}/ ${code_name} main
+deb-src [signed-by=${keyring_path}] ${bintray_base_url}/ ${code_name} main
+APT_LINE
+else
+ curl https://dist.apache.org/repos/dist/dev/arrow/KEYS | apt-key add -
+ cat <<APT_LINE > /etc/apt/sources.list.d/apache-arrow.list
+deb [arch=amd64] ${bintray_base_url}/ ${code_name} main
+deb-src ${bintray_base_url}/ ${code_name} main
+APT_LINE
+fi
+
+if [ "${need_llvm_apt}" = "yes" ]; then
+ curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+ cat <<APT_LINE > /etc/apt/sources.list.d/llvm.list
+deb http://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-7 main
+deb-src http://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-7 main
+APT_LINE
+fi
+
+apt update
+
+apt install -y -V libarrow-glib-dev=${deb_version}
+apt install -y -V libarrow-glib-doc=${deb_version}
+
+if [ "${have_python}" = "yes" ]; then
+ apt install -y -V libarrow-python-dev=${deb_version}
+fi
+
+apt install -y -V libplasma-glib-dev=${deb_version}
+apt install -y -V libplasma-glib-doc=${deb_version}
+# apt install -y -V plasma-store-server=${deb_version}
+
+if [ "${have_gandiva}" = "yes" ]; then
+ apt install -y -V libgandiva-glib-dev=${deb_version}
+ apt install -y -V libgandiva-glib-doc=${deb_version}
+fi
+
+apt install -y -V libparquet-glib-dev=${deb_version}
+apt install -y -V libparquet-glib-doc=${deb_version}
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 3b558d0..edd8f10 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -48,7 +48,7 @@ esac
set -ex
set -o pipefail
-HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
ARROW_BOOST_VENDORED=${ARROW_BOOST_VENDORED:=OFF}
@@ -117,6 +117,7 @@ download_bintray_files() {
local version_name=${VERSION}-rc${RC_NUMBER}
+ local file
bintray \
GET /packages/${BINTRAY_REPOSITORY}/${target}-rc/versions/${version_name}/files | \
jq -r ".[].path" | \
@@ -126,11 +127,12 @@ download_bintray_files() {
--fail \
--location \
--output ${file} \
- https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file}
+ https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file} &
done
+ wait
}
-verify_binary_artifacts() {
+test_binary() {
local download_dir=binaries
mkdir -p ${download_dir}
pushd ${download_dir}
@@ -159,6 +161,39 @@ verify_binary_artifacts() {
popd
}
+test_apt() {
+ for target in debian-stretch \
+ ubuntu-trusty \
+ ubuntu-xenial \
+ ubuntu-bionic \
+ ubuntu-cosmic; do
+ if ! "${SOURCE_DIR}/../run_docker_compose.sh" \
+ "${target}" \
+ /arrow/dev/release/verify-apt.sh \
+ "${VERSION}" \
+ "yes" \
+ "${BINTRAY_REPOSITORY}"; then
+ echo "Failed to verify the APT repository for ${target}"
+ exit 1
+ fi
+ done
+}
+
+test_yum() {
+ for target in centos-6 \
+ centos-7; do
+ if ! "${SOURCE_DIR}/../run_docker_compose.sh" \
+ "${target}" \
+ /arrow/dev/release/verify-yum.sh \
+ "${VERSION}" \
+ "yes" \
+ "${BINTRAY_REPOSITORY}"; then
+ echo "Failed to verify the Yum repository for ${target}"
+ exit 1
+ fi
+ done
+}
+
setup_tempdir() {
cleanup() {
rm -fr "$TMPDIR"
@@ -379,30 +414,33 @@ fi
import_gpg_keys
+# By default test all functionalities.
+# To deactivate one test, deactivate the test and all of its dependents
+# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
+: ${TEST_DEFAULT:=1}
+: ${TEST_JAVA:=${TEST_DEFAULT}}
+: ${TEST_CPP:=${TEST_DEFAULT}}
+: ${TEST_GLIB:=${TEST_DEFAULT}}
+: ${TEST_RUBY:=${TEST_DEFAULT}}
+: ${TEST_PYTHON:=${TEST_DEFAULT}}
+: ${TEST_JS:=${TEST_DEFAULT}}
+: ${TEST_INTEGRATION:=${TEST_DEFAULT}}
+: ${TEST_RUST:=${TEST_DEFAULT}}
+: ${TEST_BINARY:=${TEST_DEFAULT}}
+: ${TEST_APT:=${TEST_DEFAULT}}
+: ${TEST_YUM:=${TEST_DEFAULT}}
+
+# Automatically test if its activated by a dependent
+TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
+TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION}))
+TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON}))
+TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION}))
+TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION}))
+
if [ "$ARTIFACT" == "source" ]; then
TARBALL=apache-arrow-$1.tar.gz
DIST_NAME="apache-arrow-${VERSION}"
- # By default test all functionalities.
- # To deactivate one test, deactivate the test and all of its dependents
- # To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
- : ${TEST_DEFAULT:=1}
- : ${TEST_JAVA:=${TEST_DEFAULT}}
- : ${TEST_CPP:=${TEST_DEFAULT}}
- : ${TEST_GLIB:=${TEST_DEFAULT}}
- : ${TEST_RUBY:=${TEST_DEFAULT}}
- : ${TEST_PYTHON:=${TEST_DEFAULT}}
- : ${TEST_JS:=${TEST_DEFAULT}}
- : ${TEST_INTEGRATION:=${TEST_DEFAULT}}
- : ${TEST_RUST:=${TEST_DEFAULT}}
-
- # Automatically test if its activated by a dependent
- TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
- TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION}))
- TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON}))
- TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION}))
- TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION}))
-
git clone https://github.com/apache/arrow-testing.git
export ARROW_TEST_DATA=$PWD/arrow-testing/data
@@ -441,7 +479,15 @@ if [ "$ARTIFACT" == "source" ]; then
else
: ${BINTRAY_REPOSITORY:=apache/arrow}
- verify_binary_artifacts
+ if [ ${TEST_BINARY} -gt 0 ]; then
+ test_binary
+ fi
+ if [ ${TEST_APT} -gt 0 ]; then
+ test_apt
+ fi
+ if [ ${TEST_YUM} -gt 0 ]; then
+ test_yum
+ fi
fi
echo 'Release candidate looks good!'
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
new file mode 100755
index 0000000..8ef38c0
--- /dev/null
+++ b/dev/release/verify-yum.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -exu
+
+if [ $# -lt 2 ]; then
+ echo "Usage: $0 X.Y.Z IS_RC"
+ echo " $0 X.Y.Z IS_RC BINTRAY_REPOSITORY"
+ echo " e.g.: $0 0.13.0 yes # Verify 0.13.0 RC"
+ echo " e.g.: $0 0.13.0 no # Verify 0.13.0"
+ echo " e.g.: $0 0.13.0 yes kou/arrow # Verify 0.13.0 RC at https://bintray.com/kou/arrow"
+ exit 1
+fi
+
+VERSION="$1"
+IS_RC="$2"
+BINTRAY_REPOSITORY="${3:-apache/arrow}"
+
+bintray_base_url="https://dl.bintray.com/${BINTRAY_REPOSITORY}/centos"
+if [ "${IS_RC}" = "yes" ]; then
+ bintray_base_url="${bintray_base_url}-rc"
+fi
+
+distribution=$(cut -d " " -f 1 /etc/redhat-release | tr "A-Z" "a-z")
+if grep -q Linux /etc/redhat-release; then
+ distribution_version=$(cut -d " " -f 4 /etc/redhat-release)
+else
+ distribution_version=$(cut -d " " -f 3 /etc/redhat-release)
+fi
+distribution_version=$(echo ${distribution_version} | sed -e 's/\..*$//g')
+
+have_glib=yes
+have_parquet=yes
+case "${distribution}-${distribution_version}" in
+ centos-6)
+ have_glib=no
+ have_parquet=no
+ ;;
+esac
+
+cat <<REPO > /etc/yum.repos.d/Apache-Arrow.repo
+[apache-arrow]
+name=Apache Arrow
+baseurl=${bintray_base_url}/\$releasever/\$basearch/
+gpgcheck=1
+enabled=1
+gpgkey=${bintray_base_url}/RPM-GPG-KEY-apache-arrow
+REPO
+
+yum install -y epel-release
+
+if [ "${have_glib}" = "yes" ]; then
+ yum install -y --enablerepo=epel arrow-glib-devel-${VERSION}
+ yum install -y --enablerepo=epel arrow-glib-doc-${VERSION}
+fi
+yum install -y --enablerepo=epel arrow-python-devel-${VERSION}
+
+if [ "${have_glib}" = "yes" ]; then
+ yum install -y --enablerepo=epel plasma-glib-devel-${VERSION}
+ yum install -y --enablerepo=epel plasma-glib-doc-${VERSION}
+else
+ yum install -y --enablerepo=epel plasma-devel-${VERSION}
+fi
+
+if [ "${have_parquet}" = "yes" ]; then
+ if [ "${have_glib}" = "yes" ]; then
+ yum install -y --enablerepo=epel parquet-glib-devel-${VERSION}
+ yum install -y --enablerepo=epel parquet-glib-doc-${VERSION}
+ else
+ yum install -y --enablerepo=epel parquet-devel-${VERSION}
+ fi
+fi
diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh
index 08cff0f..6c2f08e 100755
--- a/dev/run_docker_compose.sh
+++ b/dev/run_docker_compose.sh
@@ -25,17 +25,17 @@ ARROW_SRC=$(realpath "${PWD}/..")
: ${DOCKER_COMPOSE_CONF:="${ARROW_SRC}/docker-compose.yml"}
docker_compose() {
- ${DOCKER_COMPOSE} -f ${DOCKER_COMPOSE_CONF} $@
+ ${DOCKER_COMPOSE} -f ${DOCKER_COMPOSE_CONF} "$@"
}
main() {
- docker_compose build $@
- docker_compose run --rm $@
+ docker_compose build "$1"
+ docker_compose run --rm "$@"
}
if [ $# -lt 1 ]; then
- echo "Usage: $0 <docker-compose-service>" >&2
+ echo "Usage: $0 <docker-compose-service> <docker-compose-run-options>" >&2
exit 1
fi
-main $@
+main "$@"
diff --git a/docker-compose.yml b/docker-compose.yml
index 13ad4f7..e9dcd23 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -497,3 +497,40 @@ services:
- spark-cache:/spark:delegated
# TODO(kszucs): hive-integration
+
+ ######################## Verification Containers ############################
+
+ debian-stretch:
+ image: debian:stretch
+ volumes:
+ - .:/arrow:delegated
+
+ ubuntu-trusty:
+ image: ubuntu:trusty
+ volumes:
+ - .:/arrow:delegated
+
+ ubuntu-xenial:
+ image: ubuntu:xenial
+ volumes:
+ - .:/arrow:delegated
+
+ ubuntu-bionic:
+ image: ubuntu:bionic
+ volumes:
+ - .:/arrow:delegated
+
+ ubuntu-cosmic:
+ image: ubuntu:cosmic
+ volumes:
+ - .:/arrow:delegated
+
+ centos-6:
+ image: centos:6
+ volumes:
+ - .:/arrow:delegated
+
+ centos-7:
+ image: centos:7
+ volumes:
+ - .:/arrow:delegated