You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/10/24 11:23:44 UTC
[hbase] branch master updated: HBASE-27438 Make release scripts can generate CHANGES.md and RELEASENOTES.md for 3.x (#4842)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 2a3d5b160a8 HBASE-27438 Make release scripts can generate CHANGES.md and RELEASENOTES.md for 3.x (#4842)
2a3d5b160a8 is described below
commit 2a3d5b160a885ad1c7ef6e0c943c6d9258a13ab2
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Oct 24 19:23:38 2022 +0800
HBASE-27438 Make release scripts can generate CHANGES.md and RELEASENOTES.md for 3.x (#4842)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
dev-support/create-release/do-release-docker.sh | 1 +
dev-support/create-release/prepend_changes.py | 39 ++++++++++++++++++++++
dev-support/create-release/prepend_releasenotes.py | 39 ++++++++++++++++++++++
dev-support/create-release/release-build.sh | 24 ++++++++++---
dev-support/create-release/release-util.sh | 39 ++++++++++++++--------
5 files changed, 124 insertions(+), 18 deletions(-)
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index b6874b3365d..e95429f6491 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -209,6 +209,7 @@ SKIP_TAG=$SKIP_TAG
RUNNING_IN_DOCKER=1
GIT_BRANCH=$GIT_BRANCH
NEXT_VERSION=$NEXT_VERSION
+PREV_VERSION=$PREV_VERSION
RELEASE_VERSION=$RELEASE_VERSION
RELEASE_TAG=$RELEASE_TAG
GIT_REF=$GIT_REF
diff --git a/dev-support/create-release/prepend_changes.py b/dev-support/create-release/prepend_changes.py
new file mode 100755
index 00000000000..7076cb91aed
--- /dev/null
+++ b/dev-support/create-release/prepend_changes.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import re
+import os
+
+if len(sys.argv) != 3:
+ print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
+ exit(1)
+
+pattern = re.compile(r'^## Release .+ - Unreleased .+$')
+with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
+ line = prev_r.readline()
+ while line:
+ if pattern.match(line):
+ break
+ line = prev_r.readline()
+ for newline in new_r:
+ w.writelines(newline)
+ while line:
+ w.writelines(line)
+ line = prev_r.readline()
+os.rename(sys.argv[2] + '.tmp', sys.argv[2])
diff --git a/dev-support/create-release/prepend_releasenotes.py b/dev-support/create-release/prepend_releasenotes.py
new file mode 100755
index 00000000000..1dac1346d72
--- /dev/null
+++ b/dev-support/create-release/prepend_releasenotes.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import re
+import os
+
+if len(sys.argv) != 3:
+ print("usage: %s <NEW_RELEASENOTES.md> <PREV_RELEASENOTES.md>" % sys.argv[0])
+ exit(1)
+
+pattern = re.compile(r'^# .+ Release Notes$')
+with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
+ line = prev_r.readline()
+ while line:
+ if pattern.match(line):
+ break
+ line = prev_r.readline()
+ for newline in new_r:
+ w.writelines(newline)
+ while line:
+ w.writelines(line)
+ line = prev_r.readline()
+os.rename(sys.argv[2] + '.tmp', sys.argv[2])
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index 80f386a66c2..d471aa931e7 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -134,7 +134,7 @@ if [[ "$1" == "tag" ]]; then
jira_fix_version="${PROJECT}-${RELEASE_VERSION}"
fi
shopt -u nocasematch
- update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}"
+ update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" "${PREV_VERSION}"
cd "${PROJECT}"
@@ -145,9 +145,22 @@ if [[ "$1" == "tag" ]]; then
# Create release version
maven_set_version "$RELEASE_VERSION"
find . -name pom.xml -exec git add {} \;
- git add RELEASENOTES.md CHANGES.md
+ # Always put CHANGES.md and RELEASENOTES.md to parent directory, so later we do not need to
+ # check their position when generating release data. We can not put them under the source code
+ # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
+ # generating src release tarball, we will reset the git repo
+ if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
+ git add RELEASENOTES.md CHANGES.md
+ git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
+ cp CHANGES.md ../
+ cp RELEASENOTES.md ../
+ else
+ # CHANGES.md is not tracked, should 3.x+
+ git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG"
+ mv CHANGES.md ../
+ mv RELEASENOTES.md ../
+ fi
- git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH"
git tag -s -m "Via create-release" "$RELEASE_TAG"
@@ -237,8 +250,9 @@ if [[ "$1" == "publish-dist" ]]; then
log "Copying release tarballs"
cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/"
- cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
- cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"
+ cp "CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
+ cp "RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"
+
shopt -s nocasematch
# Generate api report only if project is hbase for now.
if [ "${PROJECT}" == "hbase" ]; then
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index 6d1dcd2ca4e..6241a8538a0 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -37,6 +37,10 @@ fi
# Maven Profiles for publishing snapshots and release to Maven Central and Dist
PUBLISH_PROFILES=("-P" "apache-release,release")
+# get the current directory, we want to use some python scripts to generate
+# CHANGES.md and RELEASENOTES.md
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
function error {
log "Error: $*" >&2
exit 1
@@ -183,7 +187,8 @@ function get_release_info {
local RC_COUNT
if [ "$REV" != 0 ]; then
local PREV_REL_REV=$((REV - 1))
- PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}"
+ PREV_VERSION=${SHORT_VERSION}.${PREV_REL_REV}
+ PREV_REL_TAG="rel/${PREV_VERSION}"
if git ls-remote --tags "$ASF_REPO" "$PREV_REL_TAG" | grep -q "refs/tags/${PREV_REL_TAG}$" ; then
RC_COUNT=0
REV=$((REV + 1))
@@ -197,13 +202,16 @@ function get_release_info {
else
REV=$((REV + 1))
NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
+ # not easy to calculate it, just leave it as empty and let users provide it
+ PREV_VERSION=""
RC_COUNT=0
fi
fi
RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")"
NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")"
- export RELEASE_VERSION NEXT_VERSION
+ PREV_VERSION="$(read_config "PREV_VERSION" "$PREV_VERSION")"
+ export RELEASE_VERSION NEXT_VERSION PREV_VERSION
RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")"
if [[ -z "${RELEASE_TAG}" ]]; then
@@ -259,6 +267,7 @@ Release details:
GIT_BRANCH: $GIT_BRANCH
RELEASE_VERSION: $RELEASE_VERSION
NEXT_VERSION: $NEXT_VERSION
+PREV_VERSION: $PREV_VERSION
RELEASE_TAG: $RELEASE_TAG $([[ "$GIT_REF" != "$RELEASE_TAG" ]] && printf "\n%s\n" "GIT_REF: $GIT_REF")
API_DIFF_TAG: $API_DIFF_TAG
ASF_USERNAME: $ASF_USERNAME
@@ -528,6 +537,7 @@ function get_jira_name {
function update_releasenotes {
local project_dir="$1"
local jira_fix_version="$2"
+ local previous_jira_fix_version="$3"
local jira_project
local timing_token
timing_token="$(start_step)"
@@ -547,11 +557,17 @@ function update_releasenotes {
sed -i -e \
"/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \
"${project_dir}/CHANGES.md" || true
+ else
+ # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist
+ curl --location --fail --silent --show-error --output ${project_dir}/CHANGES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/CHANGES.md"
fi
if [ -f "${project_dir}/RELEASENOTES.md" ]; then
sed -i -e \
"/^# ${jira_project} ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project} ${jira_fix_version} Release Notes/d;}" \
"${project_dir}/RELEASENOTES.md" || true
+ else
+ # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist
+ curl --location --fail --silent --show-error --output ${project_dir}/RELEASENOTES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/RELEASENOTES.md"
fi
# Yetus will not generate CHANGES if no JIRAs fixed against the release version
@@ -566,21 +582,12 @@ function update_releasenotes {
# The releasedocmaker call above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md.
if [ -f "${project_dir}/CHANGES.md" ]; then
- # To insert into project's CHANGES.md...need to cut the top off the
- # CHANGELOG.X.X.X.md file removing license and first line and then
- # insert it after the license comment closing where we have a
- # DO NOT REMOVE marker text!
- sed -i -e '/## Release/,$!d' "${changelog}"
- sed -i -e '2,${/^# HBASE Changelog/d;}' "${project_dir}/CHANGES.md"
- sed -i -e "/DO NOT REMOVE/r ${changelog}" "${project_dir}/CHANGES.md"
+ $SELF/prepend_changes.py "${changelog}" "${project_dir}/CHANGES.md"
else
mv "${changelog}" "${project_dir}/CHANGES.md"
fi
if [ -f "${project_dir}/RELEASENOTES.md" ]; then
- # Similar for RELEASENOTES but slightly different.
- sed -i -e '/Release Notes/,$!d' "${releasenotes}"
- sed -i -e '2,${/^# RELEASENOTES/d;}' "${project_dir}/RELEASENOTES.md"
- sed -i -e "/DO NOT REMOVE/r ${releasenotes}" "${project_dir}/RELEASENOTES.md"
+ $SELF/prepend_releasenotes.py "${releasenotes}" "${project_dir}/RELEASENOTES.md"
else
mv "${releasenotes}" "${project_dir}/RELEASENOTES.md"
fi
@@ -740,3 +747,9 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
function get_host_os() {
uname -s | tr '[:lower:]' '[:upper:]'
}
+
+function is_tracked() {
+ local file=$1
+ git ls-files --error-unmatch "$file" &>/dev/null
+ return $?
+}