You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2020/05/23 03:36:35 UTC
[hbase] 02/02: HBASE-24297 release scripts should be able to use a
custom git repo
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a9fefd7f533cf9bdb6369d7159ab0df3e01b357d
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Wed May 6 14:34:55 2020 -0500
HBASE-24297 release scripts should be able to use a custom git repo
* adds a optional -r [repo] arg
* if the passed repo is on the local filesystem, creates a container mount
* when cloning a local repo configure git to share objects with the local repo instead of copying
* when cloning a local repo in a container configure the clone to have a remote that will work back on the host.
closes #1725
Signed-off-by: stack <st...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
dev-support/create-release/do-release-docker.sh | 60 ++++++++++++++++++++++++-
dev-support/create-release/release-build.sh | 14 +++---
dev-support/create-release/release-util.sh | 37 +++++++++++++++
3 files changed, 102 insertions(+), 9 deletions(-)
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index fade629..568ada7 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}"
SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=SCRIPTDIR/release-util.sh
. "$SELF/release-util.sh"
+ORIG_PWD="$(pwd)"
function usage {
local NAME
@@ -71,6 +72,7 @@ Options:
-j [path] path to local JDK installation to use building. By default the script will
use openjdk8 installed in the docker image.
-p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+ -r [repo] git repo to use for remote git operations. defaults to ASF gitbox for project.
-s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
If none specified, runs tag, then publish-dist, and then publish-release.
'publish-snapshot' is also an allowed, less used, option.
@@ -82,13 +84,15 @@ WORKDIR=
IMGTAG=latest
JAVA=
RELEASE_STEP=
-while getopts "d:fhj:p:s:t:" opt; do
+GIT_REPO=
+while getopts "d:fhj:p:r:s:t:" opt; do
case $opt in
d) WORKDIR="$OPTARG" ;;
f) DRY_RUN=0 ;;
t) IMGTAG="$OPTARG" ;;
j) JAVA="$OPTARG" ;;
p) PROJECT="$OPTARG" ;;
+ r) GIT_REPO="$OPTARG" ;;
s) RELEASE_STEP="$OPTARG" ;;
h) usage ;;
?) error "Invalid option. Run with -h for help." ;;
@@ -169,11 +173,65 @@ if [ -n "$JAVA" ]; then
JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
fi
+#TODO some debug output would be good here
+GIT_REPO_MOUNT=()
+if [ -n "${GIT_REPO}" ]; then
+ case "${GIT_REPO}" in
+ # skip the easy to identify remote protocols
+ ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
+ # for sure local
+ /*)
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # on the host but normally git wouldn't use the local optimization
+ file://*)
+ echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # have to decide if it's a local path or the "scp-ish" remote
+ *)
+ declare colon_remove_prefix;
+ declare slash_remove_prefix;
+ declare local_path;
+ colon_remove_prefix="${GIT_REPO#*:}"
+ slash_remove_prefix="${GIT_REPO#*/}"
+ if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
+ # if there was no colon at all, we assume this must be a local path
+ local_path="no colon at all"
+ elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
+ # if there was a colon and there is no slash, then we assume it must be scp-style host
+ # and a relative path
+
+ if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
+ # Given the substrings made by removing everything up to the first colon and slash
+ # we can determine which comes first based on the longer substring length.
+ # if the slash is first, then we assume the colon is part of a path name and if the colon
+ # is first then it is the seperator between a scp-style host name and the path.
+ local_path="slash happened before a colon"
+ fi
+ fi
+ if [ -n "${local_path}" ]; then
+ # convert to an absolute path
+ GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ fi
+ ;;
+ esac
+ echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+fi
+
echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
cmd=(docker run -ti \
--env-file "$ENVFILE" \
--volume "$WORKDIR:/opt/hbase-rm" \
"${JAVA_VOL[@]}" \
+ "${GIT_REPO_MOUNT[@]}" \
"hbase-rm:$IMGTAG")
echo "${cmd[*]}"
"${cmd[@]}"
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index ef79d65..86cca65 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -111,12 +111,11 @@ if [[ "$1" == "tag" ]]; then
set -o pipefail
set -x # detailed logging during action
check_get_passwords ASF_PASSWORD
- check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \
- GIT_EMAIL GIT_NAME GIT_BRANCH
- ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
- encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")"
- encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")"
- git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}"
+ check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
+ if [ -z "${GIT_REPO}" ]; then
+ check_needed_vars ASF_USERNAME ASF_PASSWORD
+ fi
+ git_clone_overwrite
# 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
# $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
@@ -186,8 +185,7 @@ fi
if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
ln -s "${PROJECT}.tag" "${PROJECT}"
else
- ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
- git clone "$ASF_REPO" "${PROJECT}"
+ git_clone_overwrite
fi
cd "${PROJECT}"
git checkout "$GIT_REF"
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index 703fb85..2297ee7 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -398,6 +398,43 @@ function configure_maven {
EOF
}
+# clone of the repo, deleting anything that exists in the working directory named after the project.
+# optionally with auth details for pushing.
+function git_clone_overwrite {
+ local asf_repo
+ if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
+ error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
+ fi
+ rm -rf "${PROJECT}"
+
+ if [[ -z "${GIT_REPO}" ]]; then
+ asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
+ echo "[INFO] clone will be of the gitbox repo for ${PROJECT}."
+ if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
+ # Ugly!
+ encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
+ encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
+ GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
+ else
+ GIT_REPO="https://${asf_repo}"
+ fi
+ else
+ echo "[INFO] clone will be of provided git repo."
+ fi
+ # N.B. we use the shared flag because the clone is short lived and if a local repo repo was
+ # given this will let us refer to objects there directly instead of hardlinks or copying.
+ # The option is silently ignored for non-local repositories. see the note on git help clone
+ # for the --shared option for details.
+ git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
+ # If this was a host local git repo then add in an alternates and remote that will
+ # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
+ # for more info see 'git help remote' and 'git help repository-layout'.
+ if [ -n "$HOST_GIT_REPO" ]; then
+ echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
+ (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
+ fi
+}
+
# Writes report into cwd!
function generate_api_report {
local project="$1"