[hudi] branch master updated: [HUDI-4005] Update release scripts to help validation (#5479)

[si...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 8c9209db28 [HUDI-4005] Update release scripts to help validation (#5479)
8c9209db28 is described below

commit 8c9209db28c10f211272bf0a18bdcb5b18f5440f
Author: Raymond Xu <27...@users.noreply.github.com>
AuthorDate: Wed May 4 07:15:54 2022 -0700

    [HUDI-4005] Update release scripts to help validation (#5479)
---
 scripts/release/deploy_staging_jars.sh     |  2 +-
 scripts/release/validate_staged_bundles.sh | 66 ++++++++++++++++++++++++++++++
 scripts/release/validate_staged_release.sh | 13 ++++--
 3 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/scripts/release/deploy_staging_jars.sh b/scripts/release/deploy_staging_jars.sh
index 6928e3600c..fa71faae80 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -43,8 +43,8 @@ declare -a ALL_VERSION_OPTS=(
 "-Dscala-2.11 -Dspark2.4 -Dflink1.13"
 "-Dscala-2.11 -Dspark2.4 -Dflink1.14"
 "-Dscala-2.12 -Dspark2.4 -Dflink1.13"
-"-Dscala-2.12 -Dspark3.1 -Dflink1.14"
 "-Dscala-2.12 -Dspark3.2 -Dflink1.14"
+"-Dscala-2.12 -Dspark3.1 -Dflink1.14" # run this last to make sure utilities bundle has spark 3.1
 )
 printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
 
diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh
new file mode 100755
index 0000000000..b210f39fbf
--- /dev/null
+++ b/scripts/release/validate_staged_bundles.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# fail immediately
+set -o errexit
+set -o nounset
+
+REPO=$1
+VERSION=$2
+
+STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi"
+
+declare -a BUNDLE_URLS=(
+"${STAGING_REPO}/hudi-datahub-sync-bundle/${VERSION}/hudi-datahub-sync-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-flink1.13-bundle_2.11/${VERSION}/hudi-flink1.13-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-flink1.13-bundle_2.12/${VERSION}/hudi-flink1.13-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-flink1.14-bundle_2.11/${VERSION}/hudi-flink1.14-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-flink1.14-bundle_2.12/${VERSION}/hudi-flink1.14-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-gcp-bundle/${VERSION}/hudi-gcp-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-hadoop-mr-bundle/${VERSION}/hudi-hadoop-mr-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-hive-sync-bundle/${VERSION}/hudi-hive-sync-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-integ-test-bundle/${VERSION}/hudi-integ-test-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-kafka-connect-bundle/${VERSION}/hudi-kafka-connect-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-presto-bundle/${VERSION}/hudi-presto-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark-bundle_2.11/${VERSION}/hudi-spark-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark-bundle_2.12/${VERSION}/hudi-spark-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark2.4-bundle_2.11/${VERSION}/hudi-spark2.4-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark2.4-bundle_2.12/${VERSION}/hudi-spark2.4-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark3-bundle_2.12/${VERSION}/hudi-spark3-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark3.1-bundle_2.12/${VERSION}/hudi-spark3.1-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-spark3.2-bundle_2.12/${VERSION}/hudi-spark3.2-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-timeline-server-bundle/${VERSION}/hudi-timeline-server-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-trino-bundle/${VERSION}/hudi-trino-bundle-${VERSION}.jar"
+"${STAGING_REPO}/hudi-utilities-bundle_2.11/${VERSION}/hudi-utilities-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-utilities-bundle_2.12/${VERSION}/hudi-utilities-bundle_2.12-${VERSION}.jar"
+"${STAGING_REPO}/hudi-utilities-slim-bundle_2.11/${VERSION}/hudi-utilities-slim-bundle_2.11-${VERSION}.jar"
+"${STAGING_REPO}/hudi-utilities-slim-bundle_2.12/${VERSION}/hudi-utilities-slim-bundle_2.12-${VERSION}.jar"
+)
+
+NOW=$(date +%s)
+TMP_DIR_FOR_BUNDLES=/tmp/${NOW}
+mkdir "$TMP_DIR_FOR_BUNDLES"
+for url in "${BUNDLE_URLS[@]}"; do
+   echo "downloading $url"
+   wget "$url" -P "$TMP_DIR_FOR_BUNDLES"
+done
+
+ls -l "$TMP_DIR_FOR_BUNDLES"
diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh
index 0e02744257..41295edfee 100755
--- a/scripts/release/validate_staged_release.sh
+++ b/scripts/release/validate_staged_release.sh
@@ -78,9 +78,8 @@ pushd $WORK_DIR
 
 # Checkout dist repo
 LOCAL_SVN_DIR=local_svn_dir
-ROOT_SVN_URL=https://dist.apache.org/repos/dist/
+ROOT_SVN_URL=https://dist.apache.org/repos/dist
 REPO_TYPE=${RELEASE_TYPE}
-#RELEASE_REPO=release
 HUDI_REPO=hudi
 
 if [ $RC_NUM == -1 ]; then
@@ -99,7 +98,15 @@ echo "Downloading from svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO}"
 (bash -c "svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} $REDIRECT") || (echo -e "\t\t Unable to checkout  ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} to $REDIRECT. Please run with --verbose to get details\n" && exit -1)
 
 echo "Validating hudi-${ARTIFACT_SUFFIX} with release type \"${REPO_TYPE}\""
-cd ${HUDI_REPO}/hudi-${ARTIFACT_SUFFIX}
+if [ $RELEASE_TYPE == "release" ]; then
+  ARTIFACT_PREFIX=
+elif [ $RELEASE_TYPE == "dev" ]; then
+  ARTIFACT_PREFIX='hudi-'
+else
+  echo "Unexpected RELEASE_TYPE: $RELEASE_TYPE"
+  exit 1;
+fi
+cd ${HUDI_REPO}/${ARTIFACT_PREFIX}${ARTIFACT_SUFFIX}
 $SHASUM hudi-${ARTIFACT_SUFFIX}.src.tgz > got.sha512
 
 echo "Checking Checksum of Source Release"