You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/10/09 05:27:18 UTC
[hbase] branch branch-2.4 updated: HBASE-26186 jenkins script for
caching artifacts should verify cached file before relying on it. (#3590)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 28e0fbc HBASE-26186 jenkins script for caching artifacts should verify cached file before relying on it. (#3590)
28e0fbc is described below
commit 28e0fbc70421912d173cc7756470a2461c9bd42e
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Sat Oct 9 00:19:43 2021 -0500
HBASE-26186 jenkins script for caching artifacts should verify cached file before relying on it. (#3590)
Signed-off-by: Michael Stack <st...@apache.org>
Signed-off-by: Duo Zhang <zh...@apache.org>
---
dev-support/Jenkinsfile | 3 +++
.../jenkins-scripts/cache-apache-project-artifact.sh | 18 +++++++++++++++---
2 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index c588acf..983938d 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -91,6 +91,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-yetus" \
--keys 'https://www.apache.org/dist/yetus/KEYS' \
+ --verify-tar-gz \
"${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
"yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
@@ -137,6 +138,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-hadoop-2" \
--keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+ --verify-tar-gz \
"${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \
"hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz"
for stale in $(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | grep -v ${HADOOP2_VERSION}); do
@@ -164,6 +166,7 @@ pipeline {
"${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--working-dir "${WORKSPACE}/downloads-hadoop-3" \
--keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+ --verify-tar-gz \
"${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \
"hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz"
for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do
diff --git a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
index 5653b05..ddd65b6 100755
--- a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
+++ b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
@@ -21,6 +21,7 @@ function usage {
echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
echo ""
echo " --force for a redownload even if /path/to/download/file.tar.gz exists."
+ echo " --verify-tar-gz Only use a cached file if it can be parsed as a gzipped tarball."
echo " --working-dir /path/to/use Path for writing tempfiles. must exist."
echo " defaults to making a directory via mktemp that we clean."
echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download."
@@ -35,6 +36,7 @@ fi
# Get arguments
declare done_if_cached="true"
+declare verify_tar_gz="false"
declare working_dir
declare cleanup="true"
declare keys
@@ -42,6 +44,7 @@ while [ $# -gt 0 ]
do
case "$1" in
--force) shift; done_if_cached="false";;
+ --verify-tar-gz) shift; verify_tar_gz="true";;
--working-dir) shift; working_dir=$1; cleanup="false"; shift;;
--keys) shift; keys=$1; shift;;
--) shift; break;;
@@ -58,9 +61,18 @@ fi
target="$1"
artifact="$2"
-if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
- echo "Reusing existing download of '${artifact}'."
- exit 0
+if [ -f "${target}" ] && [ -s "${target}" ] && [ -r "${target}" ] && [ "true" = "${done_if_cached}" ]; then
+ if [ "false" = "${verify_tar_gz}" ]; then
+ echo "Reusing existing download of '${artifact}'."
+ exit 0
+ fi
+ if ! tar tzf "${target}" > /dev/null 2>&1; then
+ echo "Cached artifact is not a well formed gzipped tarball; clearing the cached file at '${target}'."
+ rm -rf "${target}"
+ else
+ echo "Reusing existing download of '${artifact}', which is a well formed gzipped tarball."
+ exit 0
+ fi
fi
if [ -z "${working_dir}" ]; then