You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2019/10/02 16:14:24 UTC
[hbase] branch master updated: HBASE-23092 Make the RM tooling in
dev-tools/create-release generic (#671)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 7ee6d59 HBASE-23092 Make the RM tooling in dev-tools/create-release generic (#671)
7ee6d59 is described below
commit 7ee6d59ef84ee2bdf4425b70ee6fd6ea8954a563
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Wed Oct 2 09:14:17 2019 -0700
HBASE-23092 Make the RM tooling in dev-tools/create-release generic (#671)
Make the scripts generic. Adds an option that allows you specify
'project'. Defaults to 'hbase' for core. Pass 'hbase-thirdparty'
or 'hbase-operator-tools' etc.
This commit includes a bunch of bugfixes and miscellaneous
that came of trying to use the scripts making RCs.
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
dev-support/create-release/README.txt | 8 +
dev-support/create-release/do-release-docker.sh | 34 ++--
dev-support/create-release/do-release.sh | 11 +-
dev-support/create-release/release-build.sh | 241 +++++++++++-------------
dev-support/create-release/release-tag.sh | 34 ++--
dev-support/create-release/release-util.sh | 155 +++++++++++----
dev-support/create-release/vote.tmpl | 10 +-
7 files changed, 288 insertions(+), 205 deletions(-)
diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt
index 2c6c7e7..0d62671 100644
--- a/dev-support/create-release/README.txt
+++ b/dev-support/create-release/README.txt
@@ -1,6 +1,14 @@
Entrance script is _do-release-docker.sh_. Requires a local docker;
for example, on mac os x, Docker for Desktop installed and running.
+For usage, pass '-h':
+
+ $ ./do-release-docker.sh -h
+
+To run a build w/o invoking docker (not recommeneded!), use
+_do_release.sh_. It does not take parameters. It will ask
+you what commands to run with taking defaults from environment.
+
Before starting the RC build, run a reconciliation of what is in
JIRA with what is in the commit log. Make sure they align and that
anomalies are explained up in JIRA.
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index d6711bd..37a5d71 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -47,11 +46,16 @@
# 1. https://github.com/apache/spark/tree/master/dev/create-release
#
set -e
-SELF=$(cd $(dirname $0) && pwd)
+
+# Set this building other hbase repos: e.g. PROJECT=hbase-operator-tools
+export PROJECT="${PROJECT:-hbase}"
+
+SELF=$(cd $(dirname "$0") && pwd)
. "$SELF/release-util.sh"
function usage {
- local NAME=$(basename $0)
+ local NAME
+ NAME="$(basename "$0")"
cat <<EOF
Usage: $NAME [options]
@@ -59,12 +63,13 @@ This script runs the release scripts inside a docker image.
Options:
- -d [path] required. working directory. output will be written to "output" in here.
- -n dry run mode. Checks and local builds, but does not upload anything.
- -t [tag] tag for the hbase-rm docker image to use for building (default: "latest").
- -j [path] path to local JDK installation to use building. By default the script will
+ -d [path] required. working directory. output will be written to "output" in here.
+ -n dry run mode. Checks and local builds, but does not upload anything.
+ -t [tag] tag for the hbase-rm docker image to use for building (default: "latest").
+ -j [path] path to local JDK installation to use building. By default the script will
use openjdk8 installed in the docker image.
- -s [step] runs a single step of the process; valid steps are: tag, build, publish. if
+ -p [project] project to build; default 'hbase'; alternatively, 'hbase-thirdparty', etc.
+ -s [step] runs a single step of the process; valid steps are: tag, build, publish. if
none specified, runs tag, then build, and then publish.
EOF
}
@@ -73,12 +78,13 @@ WORKDIR=
IMGTAG=latest
JAVA=
RELEASE_STEP=
-while getopts "d:hj:ns:t:" opt; do
+while getopts "d:hj:np:s:t:" opt; do
case $opt in
d) WORKDIR="$OPTARG" ;;
n) DRY_RUN=1 ;;
t) IMGTAG="$OPTARG" ;;
j) JAVA="$OPTARG" ;;
+ p) PROJECT="$OPTARG" ;;
s) RELEASE_STEP="$OPTARG" ;;
h) usage ;;
?) error "Invalid option. Run with -h for help." ;;
@@ -90,7 +96,7 @@ if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
fi
if [ -d "$WORKDIR/output" ]; then
- read -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
+ read -r -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
if [ "$ANSWER" != "y" ]; then
error "Exiting."
fi
@@ -112,7 +118,7 @@ done
GPG_KEY_FILE="$WORKDIR/gpg.key"
fcreate_secure "$GPG_KEY_FILE"
-$GPG --passphrase $GPG_PASSPHRASE --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
+$GPG --passphrase "$GPG_PASSPHRASE" --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
docker build -t "hbase-rm:$IMGTAG" --build-arg UID=$UID "$SELF/hbase-rm"
@@ -129,7 +135,7 @@ function cleanup {
trap cleanup EXIT
-cat > $ENVFILE <<EOF
+cat > "$ENVFILE" <<EOF
DRY_RUN=$DRY_RUN
SKIP_TAG=$SKIP_TAG
RUNNING_IN_DOCKER=1
@@ -138,7 +144,7 @@ NEXT_VERSION=$NEXT_VERSION
RELEASE_VERSION=$RELEASE_VERSION
RELEASE_TAG=$RELEASE_TAG
GIT_REF=$GIT_REF
-HBASE_PACKAGE_VERSION=$HBASE_PACKAGE_VERSION
+PACKAGE_VERSION=$PACKAGE_VERSION
ASF_USERNAME=$ASF_USERNAME
GIT_NAME=$GIT_NAME
GIT_EMAIL=$GIT_EMAIL
@@ -152,7 +158,7 @@ EOF
JAVA_VOL=
if [ -n "$JAVA" ]; then
- echo "JAVA_HOME=/opt/hbase-java" >> $ENVFILE
+ echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
JAVA_VOL="--volume $JAVA:/opt/hbase-java"
fi
diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh
index 21c5299..58a6d8d 100755
--- a/dev-support/create-release/do-release.sh
+++ b/dev-support/create-release/do-release.sh
@@ -17,8 +17,12 @@
# limitations under the License.
#
-# Called by do-release-docker.sh. Can be run standalone but needs some love
-# for it to work smooth.
+# Use the adjacent do-release-docker.sh instead, if you can.
+# Otherwise, this runs core of the release creation.
+# Will ask you questions on what to build and for logins
+# and passwords to use building.
+export PROJECT="${PROJECT:-hbase}"
+
SELF=$(cd $(dirname $0) && pwd)
. "$SELF/release-util.sh"
@@ -30,6 +34,7 @@ while getopts "bn" opt; do
esac
done
+# If running in docker, import and then cache keys.
if [ "$RUNNING_IN_DOCKER" = "1" ]; then
# Run gpg agent.
eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
@@ -66,7 +71,7 @@ else
fi
if should_build "build"; then
- run_silent "Building HBase..." "build.log" \
+ run_silent "Building ${PROJECT}..." "build.log" \
"$SELF/release-build.sh" build
else
echo "Skipping build step."
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index ff1440a..4833a98 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -16,32 +16,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
trap cleanup EXIT
+# Source in utils.
SELF=$(cd $(dirname $0) && pwd)
. "$SELF/release-util.sh"
+# Print usage and exit.
function exit_with_usage {
cat << EOF
-usage: release-build.sh <package|docs|publish-snapshot|publish-release>
-Creates build deliverables from an HBase commit.
-
-Top level targets are
- build: Create binary packages and commit them to dist.apache.org/repos/dist/dev/hbase/
- publish-snapshot: Publish snapshot release to Apache snapshots
- publish-release: Publish a release to Apache release repo
-
-All other inputs are environment variables
-
-GIT_REF - Release tag or commit to build from
-HBASE_PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2RC1)
-HBASE_VERSION - (optional) Version of HBase being built (e.g. 2.1.2)
-
-ASF_USERNAME - Username of ASF committer account
-ASF_PASSWORD - Password of ASF committer account
-
-GPG_KEY - GPG key used to sign release artifacts
-GPG_PASSPHRASE - Passphrase for GPG key
+Usage: release-build.sh <build|publish-snapshot|publish-release>
+Creates build deliverables from a tag/commit.
+Arguments:
+ build Create binary packages and commit to dist.apache.org/repos/dist/dev/hbase/
+ publish-snapshot Publish snapshot release to Apache snapshots
+ publish-release Publish a release to Apache release repo
+
+All other inputs are environment variables:
+ GIT_REF - Release tag or commit to build from
+ PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2RC1)
+ VERSION - (optional) Version of project being built (e.g. 2.1.2)
+ ASF_USERNAME - Username of ASF committer account
+ ASF_PASSWORD - Password of ASF committer account
+ GPG_KEY - GPG key used to sign release artifacts
+ GPG_PASSPHRASE - Passphrase for GPG key
+ PROJECT - The project to build. No default.
+
+Set REPO environment to full path to repo to use
+to avoid re-downloading dependencies on each run.
+
+For example:
+ $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh build
EOF
exit 1
}
@@ -60,12 +66,14 @@ if [[ $@ == *"help"* ]]; then
exit_with_usage
fi
+# Read in the ASF password.
if [[ -z "$ASF_PASSWORD" ]]; then
echo 'The environment variable ASF_PASSWORD is not set. Enter the password.'
echo
stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
fi
+# Read in the GPG passphrase
if [[ -z "$GPG_PASSPHRASE" ]]; then
echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to'
echo 'unlock the GPG signing key that will be used to sign the release!'
@@ -92,7 +100,7 @@ RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase"
GPG="gpg --pinentry-mode loopback -u $GPG_KEY --no-tty --batch"
NEXUS_ROOT=https://repository.apache.org/service/local/staging
-NEXUS_PROFILE=8e226b97c0c82 # Profile for HBase staging uploads via INFRA-17900 Need nexus "staging profile id" for the hbase project
+NEXUS_PROFILE=8e226b97c0c82 # Profile for project staging uploads via INFRA-17900 Need nexus "staging profile id" for the hbase project
BASE_DIR=$(pwd)
init_java
@@ -101,42 +109,25 @@ init_python
# Print out subset of perl version.
perl --version | grep 'This is'
-rm -rf hbase
+rm -rf ${PROJECT}
+ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
git clone "$ASF_REPO"
-cd hbase
+cd ${PROJECT}
git checkout $GIT_REF
git_hash=`git rev-parse --short HEAD`
-echo "Checked out HBase git hash $git_hash"
+echo "Checked out ${PROJECT} git hash $git_hash"
-if [ -z "$HBASE_VERSION" ]; then
+if [ -z "$VERSION" ]; then
# Run $MVN in a separate command so that 'set -e' does the right thing.
TMP=$(mktemp)
$MVN help:evaluate -Dexpression=project.version > $TMP
- HBASE_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download)
+ VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download)
rm $TMP
fi
# Profiles for publishing snapshots and release to Maven Central
PUBLISH_PROFILES="-Papache-release -Prelease"
-if [[ ! $HBASE_VERSION < "2.0." ]]; then
- if [[ $JAVA_VERSION < "1.8." ]]; then
- echo "Java version $JAVA_VERSION is less than required 1.8 for 2.0+"
- echo "Please set JAVA_HOME correctly."
- exit 1
- fi
-else
- if ! [[ $JAVA_VERSION =~ 1\.7\..* ]]; then
- if [ -z "$JAVA_7_HOME" ]; then
- echo "Java version $JAVA_VERSION is higher than required 1.7 for pre-2.0"
- echo "Please set JAVA_HOME correctly."
- exit 1
- else
- export JAVA_HOME="$JAVA_7_HOME"
- fi
- fi
-fi
-
# This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins
# machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion
# in its pull request.
@@ -145,16 +136,16 @@ if ! hash $LSOF 2>/dev/null; then
LSOF=/usr/sbin/lsof
fi
-if [ -z "$HBASE_PACKAGE_VERSION" ]; then
- HBASE_PACKAGE_VERSION="${HBASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}"
+if [ -z "$PACKAGE_VERSION" ]; then
+ PACKAGE_VERSION="${VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}"
fi
-DEST_DIR_NAME="$HBASE_PACKAGE_VERSION"
+DEST_DIR_NAME="$PACKAGE_VERSION"
git clean -d -f -x
cd ..
-tmp_repo=`pwd`/$(mktemp -d hbase-repo-XXXXX)
+tmp_repo="${REPO:-`pwd`/$(mktemp -d hbase-repo-XXXXX)}"
# Reexamine. Not sure this working. Pass as arg? That don't seem to work either!
tmp_settings="/${tmp_repo}/tmp-settings.xml"
echo "<settings><servers>" > $tmp_settings
@@ -169,61 +160,14 @@ export tmp_settings
if [[ "$1" == "build" ]]; then
# Source and binary tarballs
echo "Packaging release source tarballs"
- # Tar up the src and sign and hash it.
- rm -rf hbase-$HBASE_VERSION-src*
- ls
- cd hbase
- git clean -d -f -x
- git archive --format=tar.gz --output=../hbase-$HBASE_VERSION-src.tar.gz --prefix=hbase-"${HBASE_VERSION}/" "${GIT_REF}"
- cd ..
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output hbase-$HBASE_VERSION-src.tar.gz.asc \
- --detach-sig hbase-$HBASE_VERSION-src.tar.gz
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \
- SHA512 hbase-$HBASE_VERSION-src.tar.gz > hbase-$HBASE_VERSION-src.tar.gz.sha512
-
+ make_src_release "${PROJECT}" "${VERSION}"
# Add timestamps to mvn logs.
MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}"
- # Updated for each binary build; spark did many. HBase does one bin only.
- make_binary_release() {
-
- echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist"
- cd hbase
-
- # Get maven home set by MVN
- MVN_HOME=`$MVN -version 2>&1 | grep 'Maven home' | awk '{print $NF}'`
-
- echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Assembly"
- git clean -d -f -x
- # Three invocations of maven. This seems to work. One to
- # populate the repo, another to build the site, and then
- # a third to assemble the binary artifact. Trying to do
- # all in the one invocation fails; a problem in our
- # assembly spec to in maven. TODO. Meantime, three invocations.
- MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \
- clean install -DskipTests \
- -Dmaven.repo.local=${tmp_repo}
- MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \
- site -DskipTests \
- -Dmaven.repo.local=${tmp_repo}
- MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \
- install assembly:single -DskipTests \
- -Dcheckstyle.skip=true ${PUBLISH_PROFILES} \
- -Dmaven.repo.local=${tmp_repo}
-
- echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Copying and signing regular binary distribution"
- cp ./hbase-assembly/target/hbase-$HBASE_VERSION*-bin.tar.gz ..
- cd ..
- for i in `ls hbase-$HBASE_VERSION*-bin.tar.gz`; do
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \
- --output $i.asc --detach-sig $i
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \
- SHA512 $i > $i.sha512
- done
- }
-
- make_binary_release
+ echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist"
+ make_binary_release "${PROJECT}" "${VERSION}"
+ echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution"
if ! is_dry_run; then
svn co --depth=empty $RELEASE_STAGING_LOCATION svn-hbase
@@ -231,17 +175,22 @@ if [[ "$1" == "build" ]]; then
mkdir -p "svn-hbase/${DEST_DIR_NAME}"
echo "Copying release tarballs"
- cp hbase-*.tar.* "svn-hbase/${DEST_DIR_NAME}/"
- cp hbase/CHANGES.md "svn-hbase/${DEST_DIR_NAME}/"
- cp hbase/RELEASENOTES.md "svn-hbase/${DEST_DIR_NAME}/"
- # This script usually reports an errcode along w/ the report.
- generate_api_report ./hbase "${API_DIFF_TAG}" "${HBASE_PACKAGE_VERSION}" || true
- cp api*.html "svn-hbase/${DEST_DIR_NAME}/"
+ cp ${PROJECT}-*.tar.* "svn-hbase/${DEST_DIR_NAME}/"
+ cp ${PROJECT}/CHANGES.md "svn-hbase/${DEST_DIR_NAME}/"
+ cp ${PROJECT}/RELEASENOTES.md "svn-hbase/${DEST_DIR_NAME}/"
+ shopt -s nocasematch
+ # Generate api report only if project is hbase for now.
+ if [ "${PROJECT}" == "hbase" ]; then
+ # This script usually reports an errcode along w/ the report.
+ generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true
+ cp api*.html "svn-hbase/${DEST_DIR_NAME}/"
+ fi
+ shopt -u nocasematch
svn add "svn-hbase/${DEST_DIR_NAME}"
cd svn-hbase
- svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache HBase $HBASE_PACKAGE_VERSION" --no-auth-cache
+ svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache
cd ..
rm -rf svn-hbase
fi
@@ -250,43 +199,54 @@ if [[ "$1" == "build" ]]; then
fi
if [[ "$1" == "publish-snapshot" ]]; then
- cd hbase
- # Publish HBase to Maven release repo
- echo "Deploying HBase SNAPSHOT at '$GIT_REF' ($git_hash)"
- echo "Publish version is $HBASE_VERSION"
- if [[ ! $HBASE_VERSION == *"SNAPSHOT"* ]]; then
+ cd "${PROJECT}"
+ # Publish ${PROJECT} to Maven release repo
+ echo "Deploying ${PROJECT} SNAPSHOT at '$GIT_REF' ($git_hash)"
+ echo "Publish version is $VERSION"
+ if [[ ! $VERSION == *"SNAPSHOT"* ]]; then
echo "ERROR: Snapshots must have a version containing SNAPSHOT"
- echo "ERROR: You gave version '$HBASE_VERSION'"
+ echo "ERROR: You gave version '$VERSION'"
exit 1
fi
# Coerce the requested version
- $MVN versions:set -DnewVersion=$HBASE_VERSION
+ $MVN versions:set -DnewVersion=$VERSION
$MVN --settings $tmp_settings -DskipTests $PUBLISH_PROFILES deploy
cd ..
exit 0
fi
if [[ "$1" == "publish-release" ]]; then
- cd hbase
- # Publish HBase to Maven release repo
- echo "Publishing HBase checkout at '$GIT_REF' ($git_hash)"
- echo "Publish version is $HBASE_VERSION"
+ cd "${PROJECT}"
+ # Get list of modules from parent pom but filter out 'assembly' modules.
+ # Used below in a few places.
+ modules=`sed -n 's/<module>\(.*\)<.*$/\1/p' pom.xml | grep -v '-assembly' | tr '\n' ' '`
+ # Need to add the 'parent' module too. Its the SECOND artifactId instance in pom
+ artifactid=`sed -n 's/<artifactId>\(.*\)<.*$/\1/p' pom.xml | tr '\n' ' '| awk '{print $2}'`
+ modules="${artifactid} ${modules}"
+ # Get the second groupId in the pom. This is the groupId for these artifacts.
+ groupid=`sed -n 's/<groupId>\(.*\)<.*$/\1/p' pom.xml | tr '\n' ' '| awk '{print $2}'`
+ # Convert groupid to a dir path for use below reaching into repo for jars.
+ groupid_as_dir=`echo $groupid | sed -n 's/\./\//gp'`
+ pwd
+ # Publish ${PROJECT} to Maven release repo
+ echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)"
+ echo "Publish version is $VERSION"
# Coerce the requested version
- $MVN versions:set -DnewVersion=$HBASE_VERSION
+ $MVN versions:set -DnewVersion=$VERSION
MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings \
clean install -DskipTests \
-Dcheckstyle.skip=true ${PUBLISH_PROFILES} \
- -Dmaven.repo.local=${tmp_repo}
- pushd $tmp_repo/org/apache/hbase
+ -Dmaven.repo.local="${tmp_repo}"
+ pushd "$tmp_repo/$groupid_as_dir"
# Remove any extra files generated during install
- # Do find in hbase* because thirdparty is at same level!
- find hbase* -type f | grep -v \.jar | grep -v \.pom | xargs rm
+ # Remove extaneous files from module subdirs
+ find $modules -type f | grep -v \.jar | grep -v \.pom | xargs rm
# Using Nexus API documented here:
# https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API
if ! is_dry_run; then
echo "Creating Nexus staging repository"
- repo_request="<promoteRequest><data><description>Apache HBase $HBASE_VERSION (commit $git_hash)</description></data></promoteRequest>"
+ repo_request="<promoteRequest><data><description>Apache ${PROJECT} $VERSION (commit $git_hash)</description></data></promoteRequest>"
out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \
-H "Content-Type:application/xml" -v \
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start)
@@ -295,39 +255,43 @@ if [[ "$1" == "publish-release" ]]; then
fi
# this must have .asc, and .sha1 - it really doesn't like anything else there
- for file in $(find hbase* -type f)
+ for file in $(find $modules -type f)
do
if [[ "$file" == *.asc ]]; then
continue
fi
if [ ! -f $file.asc ]; then
- echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --output $file.asc \
+ echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --output "$file.asc" \
--detach-sig --armour $file;
fi
if [ $(command -v md5) ]; then
# Available on OS X; -q to keep only hash
- md5 -q $file > $file.md5
+ md5 -q "$file" > "$file.md5"
else
# Available on Linux; cut to keep only hash
- md5sum $file | cut -f1 -d' ' > $file.md5
+ md5sum "$file" | cut -f1 -d' ' > "$file.md5"
+ fi
+ if [ $(command -v sha1sum) ]; then
+ sha1sum "$file" | cut -f1 -d' ' > "$file.sha1"
+ else
+ shasum "$file" | cut -f1 -d' ' > "$file.sha1"
fi
- sha1sum $file | cut -f1 -d' ' > $file.sha1
done
if ! is_dry_run; then
nexus_upload=$NEXUS_ROOT/deployByRepositoryId/$staged_repo_id
echo "Uplading files to $nexus_upload"
- for file in $(find hbase* -type f)
+ for file in $(find "${modules}" -type f)
do
# strip leading ./
file_short=$(echo $file | sed -e "s/\.\///")
- dest_url="$nexus_upload/org/apache/hbase/$file_short"
+ dest_url="$nexus_upload/$groupid_as_dir/$file_short"
echo " Uploading $file to $dest_url"
- curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $file_short $dest_url
+ curl -u "$ASF_USERNAME:$ASF_PASSWORD" --upload-file "${file_short}" "${dest_url}"
done
echo "Closing nexus staging repository"
- repo_request="<promoteRequest><data><stagedRepositoryId>$staged_repo_id</stagedRepositoryId><description>Apache HBase $HBASE_VERSION (commit $git_hash)</description></data></promoteRequest>"
+ repo_request="<promoteRequest><data><stagedRepositoryId>$staged_repo_id</stagedRepositoryId><description>Apache ${PROJECT} $VERSION (commit $git_hash)</description></data></promoteRequest>"
out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \
-H "Content-Type:application/xml" -v \
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish)
@@ -335,13 +299,20 @@ if [[ "$1" == "publish-release" ]]; then
fi
popd
- rm -rf $tmp_repo
+ rm -rf "$tmp_repo"
cd ..
- # Dump out email to send.
- eval "echo \"$(< ../vote.tmpl)\"" |tee vote.txt
+ # Dump out email to send. Where we find vote.tmpl depends
+ # on where this script is run from
+ export PROJECT_TEXT=$(echo "${PROJECT}" | sed "s/-/ /g")
+ if test -f ../vote.tmpl ; then
+ eval "echo \"$(< ../vote.tmpl)\"" |tee vote.txt
+ else
+ # Presume in cwd.
+ eval "echo \"$(< ./vote.tmpl)\"" |tee vote.txt
+ fi
exit 0
fi
cd ..
-rm -rf hbase
+rm -rf "${PROJECT}"
echo "ERROR: expects to be called with 'install', 'publish-release' or 'publish-snapshot'"
diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh
index 166fb31..98be22f 100755
--- a/dev-support/create-release/release-tag.sh
+++ b/dev-support/create-release/release-tag.sh
@@ -25,7 +25,7 @@ function exit_with_usage {
local NAME=$(basename $0)
cat << EOF
usage: $NAME
-Tags an HBase release on a particular branch.
+Tags an $PROJECT release on a particular branch.
Inputs are specified with the following environment variables:
ASF_USERNAME - Apache Username
@@ -63,13 +63,25 @@ done
init_java
init_mvn
-ASF_HBASE_REPO="gitbox.apache.org/repos/asf/hbase.git"
-
-rm -rf hbase
-git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_HBASE_REPO" -b $GIT_BRANCH
-update_releasenotes `pwd`/hbase "$RELEASE_VERSION"
+rm -rf ${PROJECT}
+
+ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
+# Ugly!
+encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
+encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
+git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH
+# NOTE: Here we are prepending project name on version for fetching
+# changes from the HBASE JIRA. It has issues for hbase, hbase-conectors,
+# hbase-operator-tools, etc.
+shopt -s nocasematch
+if [ "${PROJECT}" != "hbase" ]; then
+ # Needs the '-' on the end.
+ prefix="${PROJECT}-"
+fi
+shopt -u nocasematch
+update_releasenotes `pwd`/${PROJECT} "${prefix}${RELEASE_VERSION}"
-cd hbase
+cd ${PROJECT}
git config user.name "$GIT_NAME"
git config user.email $GIT_EMAIL
@@ -78,7 +90,7 @@ git config user.email $GIT_EMAIL
$MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs
git add RELEASENOTES.md CHANGES.md
-git commit -a -m "Preparing HBase release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
+git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH"
git tag $RELEASE_TAG
@@ -92,10 +104,10 @@ if ! is_dry_run; then
git push origin $RELEASE_TAG
git push origin HEAD:$GIT_BRANCH
cd ..
- rm -rf hbase
+ rm -rf ${PROJECT}
else
cd ..
- mv hbase hbase.tag
- echo "Clone with version changes and tag available as hbase.tag in the output directory."
+ mv ${PROJECT} ${PROJECT}.tag
+ echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory."
fi
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index e4646d1..be862ad 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -18,10 +18,6 @@
#
DRY_RUN=${DRY_RUN:-0}
GPG="gpg --pinentry-mode loopback --no-tty --batch"
-ASF_REPO="https://gitbox.apache.org/repos/asf/hbase.git"
-
-ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=hbase.git"
-ASF_GITHUB_REPO="https://github.com/apache/hbase"
YETUS_VERSION=0.9.0
function error {
@@ -54,7 +50,7 @@ function run_silent {
echo "========================"
echo "= $BANNER"
- echo "Command: $@"
+ echo "Command: $*"
echo "Log file: $LOG_FILE"
"$@" 1>"$LOG_FILE" 2>&1
@@ -95,13 +91,24 @@ function get_api_diff_version {
api_diff_tag="rel/$((major - 1)).0.0)"
fi
fi
- api_diff_tag=$(read_config "API diff tag", "$api_diff_tag")
+ api_diff_tag=$(read_config "api_diff_tag", "$api_diff_tag")
echo $api_diff_tag
}
# Get all branches that begin with 'branch-', the hbase convention for
# release branches, sort them and then pop off the most recent.
function get_release_info {
+ export PROJECT=$(read_config "PROJECT" "$PROJECT")
+
+ if [[ -z "${ASF_REPO}" ]]; then
+ ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git"
+ fi
+ if [[ -z "${ASF_REPO_WEBUI}" ]]; then
+ ASF_REPO_WEBUI="https://gitbox.apache.org/repos/asf?p=${PROJECT}.git"
+ fi
+ if [[ -z "${ASF_GITHUB_REPO}" ]]; then
+ ASF_GITHUB_REPO="https://github.com/apache/${PROJECT}"
+ fi
if [ -z "$GIT_BRANCH" ]; then
# If no branch is specified, find out the latest branch from the repo.
GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" |
@@ -112,7 +119,7 @@ function get_release_info {
cut -d/ -f3)
fi
- export GIT_BRANCH=$(read_config "Branch" "$GIT_BRANCH")
+ export GIT_BRANCH=$(read_config "GIT_BRANCH" "$GIT_BRANCH")
# Find the current version for the branch.
local VERSION=$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
@@ -153,11 +160,11 @@ function get_release_info {
RC_COUNT=0
fi
- export NEXT_VERSION
- export RELEASE_VERSION=$(read_config "Release" "$RELEASE_VERSION")
+ export RELEASE_VERSION=$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")
+ export NEXT_VERSION=$(read_config "NEXT_VERSION" "$NEXT_VERSION")
- RC_COUNT=$(read_config "RC #" "$RC_COUNT")
+ RC_COUNT=$(read_config "RC_COUNT" "$RC_COUNT")
# Check if the RC already exists, and if re-creating the RC, skip tag creation.
RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}"
@@ -175,35 +182,34 @@ function get_release_info {
GIT_REF="$RELEASE_TAG"
if is_dry_run; then
echo "This is a dry run. Please confirm the ref that will be built for testing."
- GIT_REF=$(read_config "Ref" "$GIT_REF")
+ GIT_REF=$(read_config "GIT_REF" "$GIT_REF")
fi
export GIT_REF
- export HBASE_PACKAGE_VERSION="$RELEASE_TAG"
+ export PACKAGE_VERSION="$RELEASE_TAG"
export API_DIFF_TAG=$(get_api_diff_version $RELEASE_VERSION)
# Gather some user information.
- export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME")
+ export ASF_USERNAME=$(read_config "ASF_USERNAME" "$LOGNAME")
GIT_NAME=$(git config user.name || echo "")
- export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+ export GIT_NAME=$(read_config "GIT_NAME" "$GIT_NAME")
export GIT_EMAIL="$ASF_USERNAME@apache.org"
- export GPG_KEY=$(read_config "GPG key" "$GIT_EMAIL")
+ export GPG_KEY=$(read_config "GPG_KEY" "$GIT_EMAIL")
cat <<EOF
================
Release details:
-BRANCH: $GIT_BRANCH
-VERSION: $RELEASE_VERSION
-TAG: $RELEASE_TAG
-NEXT: $NEXT_VERSION
-API DIFF TAG: $API_DIFF_TAG
-
-ASF USER: $ASF_USERNAME
-GPG KEY: $GPG_KEY
-FULL NAME: $GIT_NAME
-E-MAIL: $GIT_EMAIL
+GIT_BRANCH: $GIT_BRANCH
+RELEASE_VERSION: $RELEASE_VERSION
+RELEASE_TAG: $RELEASE_TAG
+NEXT_VERSION: $NEXT_VERSION
+API_DIFF_TAG: $API_DIFF_TAG
+ASF_USERNAME: $ASF_USERNAME
+GPG_KEY: $GPG_KEY
+GIT_NAME: $GIT_NAME
+GIT-EMAIL: $GIT_EMAIL
================
EOF
@@ -215,14 +221,14 @@ EOF
if ! is_dry_run; then
if [ -z "$ASF_PASSWORD" ]; then
- stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
+ stty -echo && printf "ASF_PASSWORD: " && read ASF_PASSWORD && printf '\n' && stty echo
fi
else
ASF_PASSWORD="***INVALID***"
fi
if [ -z "$GPG_PASSPHRASE" ]; then
- stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo
+ stty -echo && printf "GPG_PASSPHRASE: " && read GPG_PASSPHRASE && printf '\n' && stty echo
export GPG_TTY=$(tty)
fi
@@ -269,43 +275,116 @@ function init_mvn {
# Writes report into cwd!
function generate_api_report {
- local hbase=$1
+ local project="$1"
local previous_tag="$2"
local release_tag="$3"
# Generate api report.
- ${hbase}/dev-support/checkcompatibility.py --annotation \
+ ${project}/dev-support/checkcompatibility.py --annotation \
org.apache.yetus.audience.InterfaceAudience.Public \
$previous_tag $release_tag
local previous_version=$(echo ${previous_tag} | sed -e 's/rel\///')
- cp ${hbase}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html"
+ cp ${project}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html"
}
# Update the CHANGES.md
# DOES NOT DO COMMITS! Caller should do that.
# yetus requires python2 to be on the path.
function update_releasenotes {
- local hbase="$1"
+ local project="$1"
local release_version="$2"
local yetus="apache-yetus-${YETUS_VERSION}"
wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \
tar xvz -C .
- cd ./${yetus}
+ cd ./${yetus} || exit
./bin/releasedocmaker -p HBASE --fileversions -v ${release_version} -l --sortorder=newer --skip-credits
# First clear out the changes written by previous RCs.
+ pwd
sed -i -e "/^## Release ${release_version}/,/^## Release/ {//!d; /^## Release ${release_version}/d;}" \
- ${hbase}/CHANGES.md
+ ${project}/CHANGES.md || true
sed -i -e "/^# HBASE ${release_version} Release Notes/,/^# HBASE/{//!d; /^# HBASE ${release_version} Release Notes/d;}" \
- ${hbase}/RELEASENOTES.md
+ ${project}/RELEASENOTES.md || true
# The above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md.
- # To insert into hbase CHANGES.me...need to cut the top off the
+ # To insert into project CHANGES.me...need to cut the top off the
# CHANGELOG.X.X.X.md file removing license and first line and then
# insert it after the license comment closing where we have a
# DO NOT REMOVE marker text!
sed -i -e '/## Release/,$!d' CHANGELOG.${release_version}.md
- sed -i -e "/DO NOT REMOVE/r CHANGELOG.${release_version}.md" ${hbase}/CHANGES.md
+ sed -i -e "/DO NOT REMOVE/r CHANGELOG.${release_version}.md" ${project}/CHANGES.md
# Similar for RELEASENOTES but slightly different.
sed -i -e '/Release Notes/,$!d' RELEASENOTES.${release_version}.md
- sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${release_version}.md" ${hbase}/RELEASENOTES.md
- cd ..
+ sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${release_version}.md" ${project}/RELEASENOTES.md
+ cd .. || exit
+}
+
+# Make src release.
+# Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
+# -- and then the version string. Expects to find checkout adjacent to this script
+# named for 'project', the first arg passed.
+# Expects the following three defines in the environment:
+# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
+# - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file).
+# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
+# For example:
+# $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
+make_src_release() {
+ # Tar up the src and sign and hash it.
+ project="${1}"
+ version="${2}"
+ basename="${project}-${version}"
+ rm -rf "${basename}-src*"
+ tgz="${basename}-src.tar.gz"
+ cd "${project}" || exit
+ git clean -d -f -x
+ git archive --format=tar.gz --output="../${tgz}" --prefix="${basename}/" "${GIT_REF:-master}"
+ cd .. || exit
+ echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "${tgz}.asc" \
+ --detach-sig "${tgz}"
+ echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${tgz}" > "${tgz}.sha512"
+}
+
+# Make binary release.
+# Takes as arguments first the project name -- e.g. hbase or hbase-operator-tools
+# -- and then the version string. Expects to find checkout adjacent to this script
+# named for 'project', the first arg passed.
+# Expects the following three defines in the environment:
+# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
+# - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file).
+# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
+# - MVN Default is 'mvn'.
+# For example:
+# $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
+make_binary_release() {
+ project="${1}"
+ version="${2}"
+ basename="${project}-${version}"
+ rm -rf "${basename}-bin*"
+ cd $project || exit
+
+ git clean -d -f -x
+ # Three invocations of maven. This seems to work. One to
+ # populate the repo, another to build the site, and then
+ # a third to assemble the binary artifact. Trying to do
+ # all in the one invocation fails; a problem in our
+ # assembly spec to in maven. TODO. Meantime, three invocations.
+ MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings clean install -DskipTests \
+ -Dmaven.repo.local="${tmp_repo}"
+ MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings site -DskipTests \
+ -Dmaven.repo.local="${tmp_repo}"
+ MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings install assembly:single -DskipTests \
+ -Dcheckstyle.skip=true "${PUBLISH_PROFILES}" -Dmaven.repo.local="${tmp_repo}"
+
+ # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty.
+ f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz"
+ if test -f "${f_bin_tgz}"; then
+ cp "${f_bin_tgz}" ..
+ cd .. || exit
+ for i in "${basename}"*-bin.tar.gz; do
+ echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i"
+ echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${i}" > "$i.sha512"
+ done
+ else
+ cd .. || exit
+ echo "No ${f_bin_tgz} product; expected?"
+ fi
}
diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl
index d2f5398..c9c4b74 100644
--- a/dev-support/create-release/vote.tmpl
+++ b/dev-support/create-release/vote.tmpl
@@ -1,13 +1,14 @@
-Please vote on this Apache HBase release candidate (RC), ${RELEASE_VERSION}.
+Please vote on this Apache ${PROJECT_TEXT} release candidate,
+${PROJECT}-${RELEASE_TAG}
The VOTE will remain open for at least 72 hours.
-[ ] +1 Release this package as Apache HBase ${RELEASE_VERSION}
+[ ] +1 Release this package as Apache ${PROJECT_TEXT} ${RELEASE_VERSION}
[ ] -1 Do not release this package because ...
The tag to be voted on is ${RELEASE_TAG}:
- https://github.com/apache/hbase/tree/${RELEASE_TAG}
+https://github.com/apache/${PROJECT}/tree/${RELEASE_TAG}
The release files, including signatures, digests, as well as CHANGES.md
and RELEASENOTES.md included in this RC can be found at:
@@ -22,7 +23,8 @@ Artifacts were signed with the ${GPG_KEY} key which can be found in:
https://dist.apache.org/repos/dist/release/hbase/KEYS
-To learn more about Apache HBase, please see http://hbase.apache.org/
+ To learn more about apache ${PROJECT_TEXT}, please see
+http://hbase.apache.org/
Thanks,
Your HBase Release Manager