You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2020/05/23 03:36:33 UTC
[hbase] branch master updated (2b894dc -> a9fefd7)
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 2b894dc HBASE-24379 CatalogJanitor misreports region holes when there are actually over laps. (#1741)
new 8ff8e70 HBASE-24296 install yetus as a part of building the rm docker image.
new a9fefd7 HBASE-24297 release scripts should be able to use a custom git repo
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/create-release/do-release-docker.sh | 61 ++++++++++++++++++++++-
dev-support/create-release/do-release.sh | 8 ++++
dev-support/create-release/hbase-rm/Dockerfile | 6 +++
dev-support/create-release/release-build.sh | 16 +++----
dev-support/create-release/release-util.sh | 64 ++++++++++++++++++++++---
5 files changed, 139 insertions(+), 16 deletions(-)
[hbase] 02/02: HBASE-24297 release scripts should be able to use a
custom git repo
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a9fefd7f533cf9bdb6369d7159ab0df3e01b357d
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Wed May 6 14:34:55 2020 -0500
HBASE-24297 release scripts should be able to use a custom git repo
* adds a optional -r [repo] arg
* if the passed repo is on the local filesystem, creates a container mount
* when cloning a local repo configure git to share objects with the local repo instead of copying
* when cloning a local repo in a container configure the clone to have a remote that will work back on the host.
closes #1725
Signed-off-by: stack <st...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
dev-support/create-release/do-release-docker.sh | 60 ++++++++++++++++++++++++-
dev-support/create-release/release-build.sh | 14 +++---
dev-support/create-release/release-util.sh | 37 +++++++++++++++
3 files changed, 102 insertions(+), 9 deletions(-)
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index fade629..568ada7 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}"
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=SCRIPTDIR/release-util.sh
. "$SELF/release-util.sh"
+ORIG_PWD="$(pwd)"
function usage {
local NAME
@@ -71,6 +72,7 @@ Options:
-j [path] path to local JDK installation to use building. By default the script will
use openjdk8 installed in the docker image.
-p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+ -r [repo] git repo to use for remote git operations. defaults to ASF gitbox for project.
-s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
If none specified, runs tag, then publish-dist, and then publish-release.
'publish-snapshot' is also an allowed, less used, option.
@@ -82,13 +84,15 @@ WORKDIR=
IMGTAG=latest
JAVA=
RELEASE_STEP=
-while getopts "d:fhj:p:s:t:" opt; do
+GIT_REPO=
+while getopts "d:fhj:p:r:s:t:" opt; do
case $opt in
d) WORKDIR="$OPTARG" ;;
f) DRY_RUN=0 ;;
t) IMGTAG="$OPTARG" ;;
j) JAVA="$OPTARG" ;;
p) PROJECT="$OPTARG" ;;
+ r) GIT_REPO="$OPTARG" ;;
s) RELEASE_STEP="$OPTARG" ;;
h) usage ;;
?) error "Invalid option. Run with -h for help." ;;
@@ -169,11 +173,65 @@ if [ -n "$JAVA" ]; then
JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
fi
+#TODO some debug output would be good here
+GIT_REPO_MOUNT=()
+if [ -n "${GIT_REPO}" ]; then
+ case "${GIT_REPO}" in
+ # skip the easy to identify remote protocols
+ ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
+ # for sure local
+ /*)
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # on the host but normally git wouldn't use the local optimization
+ file://*)
+ echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # have to decide if it's a local path or the "scp-ish" remote
+ *)
+ declare colon_remove_prefix;
+ declare slash_remove_prefix;
+ declare local_path;
+ colon_remove_prefix="${GIT_REPO#*:}"
+ slash_remove_prefix="${GIT_REPO#*/}"
+ if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
+ # if there was no colon at all, we assume this must be a local path
+ local_path="no colon at all"
+ elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
+ # if there was a colon and there is no slash, then we assume it must be scp-style host
+ # and a relative path
+
+ if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
+ # Given the substrings made by removing everything up to the first colon and slash
+ # we can determine which comes first based on the longer substring length.
+ # if the slash is first, then we assume the colon is part of a path name and if the colon
+ # is first then it is the seperator between a scp-style host name and the path.
+ local_path="slash happened before a colon"
+ fi
+ fi
+ if [ -n "${local_path}" ]; then
+ # convert to an absolute path
+ GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ fi
+ ;;
+ esac
+ echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+fi
+
echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
cmd=(docker run -ti \
--env-file "$ENVFILE" \
--volume "$WORKDIR:/opt/hbase-rm" \
"${JAVA_VOL[@]}" \
+ "${GIT_REPO_MOUNT[@]}" \
"hbase-rm:$IMGTAG")
echo "${cmd[*]}"
"${cmd[@]}"
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index ef79d65..86cca65 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -111,12 +111,11 @@ if [[ "$1" == "tag" ]]; then
set -o pipefail
set -x # detailed logging during action
check_get_passwords ASF_PASSWORD
- check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \
- GIT_EMAIL GIT_NAME GIT_BRANCH
- ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
- encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")"
- encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")"
- git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}"
+ check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
+ if [ -z "${GIT_REPO}" ]; then
+ check_needed_vars ASF_USERNAME ASF_PASSWORD
+ fi
+ git_clone_overwrite
# 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
# $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
@@ -186,8 +185,7 @@ fi
if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
ln -s "${PROJECT}.tag" "${PROJECT}"
else
- ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
- git clone "$ASF_REPO" "${PROJECT}"
+ git_clone_overwrite
fi
cd "${PROJECT}"
git checkout "$GIT_REF"
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index 703fb85..2297ee7 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -398,6 +398,43 @@ function configure_maven {
EOF
}
+# clone of the repo, deleting anything that exists in the working directory named after the project.
+# optionally with auth details for pushing.
+function git_clone_overwrite {
+ local asf_repo
+ if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
+ error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
+ fi
+ rm -rf "${PROJECT}"
+
+ if [[ -z "${GIT_REPO}" ]]; then
+ asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
+ echo "[INFO] clone will be of the gitbox repo for ${PROJECT}."
+ if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
+ # Ugly!
+ encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
+ encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
+ GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
+ else
+ GIT_REPO="https://${asf_repo}"
+ fi
+ else
+ echo "[INFO] clone will be of provided git repo."
+ fi
+ # N.B. we use the shared flag because the clone is short lived and if a local repo repo was
+ # given this will let us refer to objects there directly instead of hardlinks or copying.
+ # The option is silently ignored for non-local repositories. see the note on git help clone
+ # for the --shared option for details.
+ git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
+ # If this was a host local git repo then add in an alternates and remote that will
+ # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
+ # for more info see 'git help remote' and 'git help repository-layout'.
+ if [ -n "$HOST_GIT_REPO" ]; then
+ echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
+ (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
+ fi
+}
+
# Writes report into cwd!
function generate_api_report {
local project="$1"
[hbase] 01/02: HBASE-24296 install yetus as a part of building the
rm docker image.
Posted by bu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8ff8e70edfbb50967cf123e6afdd9cc5dcfa4878
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Thu May 7 13:31:31 2020 -0500
HBASE-24296 install yetus as a part of building the rm docker image.
* non-zero exit for usage
* in non-docker mode prompt for yetus install location
* make sure we exit if YETUS_HOME does not actually point at a yetus install.
closes #1726
Signed-off-by: Matt Foley <ma...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
Signed-off-by: stack <st...@apache.org>
---
dev-support/create-release/do-release-docker.sh | 1 +
dev-support/create-release/do-release.sh | 8 ++++++++
dev-support/create-release/hbase-rm/Dockerfile | 6 ++++++
dev-support/create-release/release-build.sh | 2 ++
dev-support/create-release/release-util.sh | 27 ++++++++++++++++++-------
5 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index 645e8f2..fade629 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -75,6 +75,7 @@ Options:
If none specified, runs tag, then publish-dist, and then publish-release.
'publish-snapshot' is also an allowed, less used, option.
EOF
+ exit 1
}
WORKDIR=
diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh
index 313d277..b359d4b 100755
--- a/dev-support/create-release/do-release.sh
+++ b/dev-support/create-release/do-release.sh
@@ -83,6 +83,14 @@ function should_build {
}
if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then
+ if [ -z "${YETUS_HOME}" ] && [ "${RUNNING_IN_DOCKER}" != "1" ]; then
+ declare local_yetus="/opt/apache-yetus/0.11.1/"
+ if [ "$(get_host_os)" = "DARWIN" ]; then
+ local_yetus="/usr/local/Cellar/yetus/0.11.1/"
+ fi
+ YETUS_HOME="$(read_config "YETUS_HOME not defined. Absolute path to local install of Apache Yetus" "${local_yetus}")"
+ export YETUS_HOME
+ fi
run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
"$SELF/release-build.sh" tag
if is_dry_run; then
diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile
index 823aff0..3b7afdb 100644
--- a/dev-support/create-release/hbase-rm/Dockerfile
+++ b/dev-support/create-release/hbase-rm/Dockerfile
@@ -43,6 +43,12 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
&& update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
&& pip install \
python-dateutil==2.8.1
+# Install Apache Yetus
+ENV YETUS_VERSION 0.11.1
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
+ tar xvz -C /opt
+ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
WORKDIR /opt/hbase-rm/output
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index ba6f3e8..ef79d65 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -55,6 +55,7 @@ Used for 'tag' and 'publish' stages:
to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh.
Used only for 'tag':
+ YETUS_HOME - installation location for Apache Yetus
GIT_NAME - Name to use with git
GIT_EMAIL - E-mail address to use with git
GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch.
@@ -105,6 +106,7 @@ perl --version | grep 'This is'
rm -rf "${PROJECT}"
if [[ "$1" == "tag" ]]; then
+ init_yetus
# for 'tag' stage
set -o pipefail
set -x # detailed logging during action
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index 0a20fea..703fb85 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -18,7 +18,6 @@
#
DRY_RUN=${DRY_RUN:-1} #default to dry run
GPG="gpg --pinentry-mode loopback --no-tty --batch"
-YETUS_VERSION=${YETUS_VERSION:-0.11.1}
# Maven Profiles for publishing snapshots and release to Maven Central and Dist
PUBLISH_PROFILES=("-P" "apache-release,release")
@@ -349,6 +348,16 @@ function init_mvn {
configure_maven
}
+function init_yetus {
+ declare YETUS_VERSION
+ if [ -z "${YETUS_HOME}" ]; then
+ error "Missing Apache Yetus."
+ fi
+ # Work around yetus bug by asking test-patch for the version instead of rdm.
+ YETUS_VERSION=$("${YETUS_HOME}/bin/test-patch" --version)
+ echo "Apache Yetus version ${YETUS_VERSION}"
+}
+
function configure_maven {
# Add timestamps to mvn logs.
MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}"
@@ -423,17 +432,15 @@ function get_jira_name {
# Update the CHANGES.md
# DOES NOT DO COMMITS! Caller should do that.
+# requires yetus to have a defined home already.
# yetus requires python2 to be on the path.
function update_releasenotes {
local project_dir="$1"
local jira_fix_version="$2"
- local yetus="apache-yetus-${YETUS_VERSION}"
local jira_project
jira_project="$(get_jira_name "$(basename "$project_dir")")"
- wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \
- tar xvz -C . || exit
- cd "./${yetus}" || exit
- ./bin/releasedocmaker -p "${jira_project}" --fileversions -v "${jira_fix_version}" -l --sortorder=newer --skip-credits
+ "${YETUS_HOME}/bin/releasedocmaker" -p "${jira_project}" --fileversions -v "${jira_fix_version}" \
+ -l --sortorder=newer --skip-credits
pwd
# First clear out the changes written by previous RCs.
if [ -f "${project_dir}/CHANGES.md" ]; then
@@ -466,7 +473,6 @@ function update_releasenotes {
else
mv "RELEASENOTES.${jira_fix_version}.md" "${project_dir}/RELEASENOTES.md"
fi
- cd .. || exit
}
# Make src release.
@@ -608,3 +614,10 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
echo "BUILD SUCCESS."
return 0
}
+
+# guess the host os
+# * DARWIN
+# * LINUX
+function get_host_os() {
+ uname -s | tr '[:lower:]' '[:upper:]'
+}