You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/10/29 12:56:48 UTC
[hbase] branch master updated: HBASE-27449 Use xmllint to parse version and revision in release scripts (#4850)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 984d2260105 HBASE-27449 Use xmllint to parse version and revision in release scripts (#4850)
984d2260105 is described below
commit 984d226010558c5b5e54d283b465a645816ffb24
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Oct 29 20:56:42 2022 +0800
HBASE-27449 Use xmllint to parse version and revision in release scripts (#4850)
Also upgrade our Dockerfile for release scripts to use ubuntu 22.04,
maven 3.8.6 and yetus 0.14.1.
Use python3 only.
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
dev-support/create-release/hbase-rm/Dockerfile | 52 +++++++++++---------
dev-support/create-release/prepend_changes.py | 2 +-
dev-support/create-release/release-build.sh | 2 +-
dev-support/create-release/release-util.sh | 67 +++++++++++++++-----------
4 files changed, 69 insertions(+), 54 deletions(-)
diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile
index c43976f61dd..10cd1a8cb78 100644
--- a/dev-support/create-release/hbase-rm/Dockerfile
+++ b/dev-support/create-release/hbase-rm/Dockerfile
@@ -15,11 +15,11 @@
# limitations under the License.
#
-# Image for building HBase releases. Based on Ubuntu 18.04.
+# Image for building HBase releases. Based on Ubuntu 22.04.
#
# Includes:
# * Java 8
-FROM ubuntu:18.04
+FROM ubuntu:22.04
# Install extra needed repos and refresh.
@@ -28,40 +28,46 @@ FROM ubuntu:18.04
# the most current package versions (instead of potentially using old versions cached by docker).
RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
&& DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
- curl='7.58.0-*' \
- git='1:2.17.1-*' \
- gnupg='2.2.4-*' \
- libcurl4-openssl-dev='7.58.0-*' \
- libxml2-dev='2.9.4+dfsg1-*' \
- lsof='4.89+dfsg-*' \
+ curl='7.81.0-*' \
+ git='1:2.34.1-*' \
+ gnupg='2.2.27-*' \
+ libcurl4-openssl-dev='7.81.0-*' \
+ libxml2-dev='2.9.13+dfsg-*' \
+ libxml2-utils='2.9.13+dfsg-*' \
+ lsof='4.93.2+dfsg-*' \
openjdk-8-jdk='8u*' \
- python-pip='9.0.1-*' \
- subversion='1.9.7-*' \
- wget='1.19.4-*' \
+ python3-pip='22.0.2+dfsg-*' \
+ subversion='1.14.1-*' \
+ wget='1.21.2-*' \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
- && pip install \
- python-dateutil==2.8.1
+ && pip3 install \
+ python-dateutil==2.8.2
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-# Install mvn 3.6.3.
-ARG MAVEN_VERSION=3.6.3
-ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0
-ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+# Install maven
+ENV MAVEN_VERSION=3.8.6
+ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
+ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26"
RUN mkdir -p /opt/maven \
- && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
- && echo "${SHA} /tmp/apache-maven.tar.gz" | sha512sum -c - \
+ && curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \
+ && echo "${MAVEN_SHA512} /tmp/apache-maven.tar.gz" | sha512sum -c - \
&& tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \
&& rm -f /tmp/apache-maven.tar.gz \
&& ln -s /opt/maven/bin/mvn /usr/bin/mvn
# Install Apache Yetus
-ENV YETUS_VERSION 0.12.0
-RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
- tar xvz -C /opt
-ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
+ENV YETUS_VERSION 0.14.1
+ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz"
+ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109"
+RUN mkdir -p /opt/yetus \
+ && curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \
+ && echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \
+ && tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \
+ && rm -f /tmp/apache-yetus.tar.gz
+ENV YETUS_HOME /opt/yetus
ARG UID
ARG RM_USER
diff --git a/dev-support/create-release/prepend_changes.py b/dev-support/create-release/prepend_changes.py
index bba51594809..7e9dfbb4e51 100755
--- a/dev-support/create-release/prepend_changes.py
+++ b/dev-support/create-release/prepend_changes.py
@@ -24,7 +24,7 @@ if len(sys.argv) != 3:
print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
sys.exit(1)
-pattern = re.compile(r'^## Release .+ - Unreleased .+$')
+pattern = re.compile(r'^## Release .+$')
with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
line = prev_r.readline()
while line:
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index d471aa931e7..fa9844a2df0 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -149,7 +149,7 @@ if [[ "$1" == "tag" ]]; then
# check their position when generating release data. We can not put them under the source code
# directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
# generating src release tarball, we will reset the git repo
- if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
+ if is_tracked "CHANGES.md"; then
git add RELEASENOTES.md CHANGES.md
git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
cp CHANGES.md ../
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index fdb078810ce..64b481ac3c2 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -60,8 +60,11 @@ function read_config {
}
function parse_version {
- grep -e '<version>.*</version>' | \
- head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
+ xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" -
+}
+
+function parse_revision {
+ xmllint --xpath "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='revision']/text()" -
}
function banner {
@@ -135,6 +138,8 @@ function get_api_diff_version {
# Get all branches that begin with 'branch-', the hbase convention for
# release branches, sort them and then pop off the most recent.
function get_release_info {
+ init_xmllint
+
PROJECT="$(read_config "PROJECT" "$PROJECT")"
export PROJECT
@@ -164,6 +169,14 @@ function get_release_info {
local version
version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
parse_version)"
+ # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+ # If we use ${revision} as placeholder, we need to parse the revision property to
+ # get maven version
+ # shellcheck disable=SC2016
+ if [[ "${version}" == '${revision}' ]]; then
+ version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
+ parse_revision)"
+ fi
log "Current branch VERSION is $version."
NEXT_VERSION="$version"
@@ -344,6 +357,17 @@ function init_locale {
export LANG="$locale_value"
}
+# Check whether xmllint is available
+function init_xmllint {
+ if ! [ -x "$(command -v xmllint)" ]; then
+ log "Error: xmllint is not available, we need to use it for parsing pom.xml." >&2
+ log "Ubuntu: apt install libxml2-utils" >&2
+ log "CentOS: yum install xmlstarlet" >&2
+ log "Mac OS: brew install xmlstarlet" >&2
+ exit 1
+ fi
+}
+
# Initializes JAVA_VERSION to the version of the JVM in use.
function init_java {
if [ -z "$JAVA_HOME" ]; then
@@ -355,10 +379,10 @@ function init_java {
}
function init_python {
- if ! [ -x "$(command -v python2)" ]; then
- error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2'
+ if ! [ -x "$(command -v python3)" ]; then
+ error 'python3 needed by yetus and api report. Install or add link?'
fi
- log "python version: $(python2 --version)"
+ log "python3 version: $(python3 --version)"
}
# Set MVN
@@ -440,8 +464,8 @@ function git_clone_overwrite {
log "Clone will be of the gitbox repo for ${PROJECT}."
if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
# Ugly!
- encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')")
- encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')")
+ encoded_username=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_USERNAME''', ''))")
+ encoded_password=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_PASSWORD''', ''))")
GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
else
GIT_REPO="https://${asf_repo}"
@@ -533,7 +557,7 @@ function get_jira_name {
# Update the CHANGES.md
# DOES NOT DO COMMITS! Caller should do that.
# requires yetus to have a defined home already.
-# yetus requires python2 to be on the path.
+# yetus requires python3 to be on the path.
function update_releasenotes {
local project_dir="$1"
local jira_fix_version="$2"
@@ -686,28 +710,13 @@ function kick_gpg_agent {
function maven_set_version { #input: <version_to_set>
local this_version="$1"
local use_revision='false'
- local -a version_splits=()
- IFS='.' read -ar version_splits <<< "$(maven_get_version)"
-
- # Do the right thing based on project and release line.
- if [ "${PROJECT}" = 'hbase' ] ; then
- if [ "${version_splits[0]}" -le 1 ] ; then
- use_revision='false'
- elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -le 4 ] ; then
- use_revision='false'
- elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -ge 5 ] ; then
- use_revision='true'
- elif [ "${version_splits[0]}" -ge 3 ] ; then
- use_revision='true'
- fi
- elif [ "${PROJECT}" = 'hbase-thirdparty' ] ; then
- use_revision='false'
- elif [ "${PROJECT}" = 'hbase-connectors' ] ; then
+ local maven_version
+ maven_version="$(parse_version < pom.xml)"
+ # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+ # If we use ${revision} as placeholder, the way to bump maven version will be different
+ # shellcheck disable=SC2016
+ if [[ "${maven_version}" == '${revision}' ]]; then
use_revision='true'
- elif [ "${PROJECT}" = 'hbase-filesystem' ] ; then
- use_revision='false'
- elif [ "${PROJECT}" = 'hbase-operator-tools' ] ; then
- use_revision='false'
fi
if [ "${use_revision}" = 'false' ] ; then