You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/06/29 13:41:59 UTC
(hbase) 01/02: HBASE-28694 Make client integration and packaging test work with java 17 (#6035)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5ed037d23d2b2c4abd55ba255906645a4b9f3bf3
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Jun 29 21:36:57 2024 +0800
HBASE-28694 Make client integration and packaging test work with java 17 (#6035)
Signed-off-by: Xin Sun <su...@apache.org>
(cherry picked from commit c722dde59fcfc8b5e176e8e92923d759bd761163)
---
dev-support/Jenkinsfile | 50 ++++++++++++++--------
.../hbase_nightly_pseudo-distributed-test.sh | 4 +-
dev-support/hbase_nightly_source-artifact.sh | 30 ++++++++-----
3 files changed, 54 insertions(+), 30 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 0a47726e53c..94652560614 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -758,11 +758,6 @@ pipeline {
label 'hbase-large'
}
}
- tools {
- maven 'maven_latest'
- // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
- jdk "jdk_1.8_latest"
- }
environment {
BASEDIR = "${env.WORKSPACE}/component"
BRANCH = "${env.BRANCH_NAME}"
@@ -797,21 +792,25 @@ pipeline {
echo "got the following saved stats in 'output-srctarball/machine'"
ls -lh "output-srctarball/machine"
'''
- sh """#!/bin/bash -e
+ sh '''#!/bin/bash -e
echo "Checking the steps for an RM to make a source artifact, then a binary artifact."
- if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" --workdir=/hbase hbase-integration-test \
+ "component/dev-support/hbase_nightly_source-artifact.sh" \
--intermediate-file-dir output-srctarball \
--unpack-temp-dir unpacked_src_tarball \
--maven-m2-initial .m2-for-repo \
--maven-m2-src-build .m2-for-src \
--clean-source-checkout \
- "${env.BASEDIR}" ; then
+ component
+ if [ $? -eq 0 ]; then
echo '(/) {color:green}+1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
else
echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
exit 1
fi
- """
+ '''
echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
sh '''#!/bin/bash -e
if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then
@@ -834,21 +833,25 @@ pipeline {
'''
unstash 'hadoop-2'
sh '''#!/bin/bash -xe
- if [[ "${BRANCH}" = branch-2* ]]; then
+ if [[ "${BRANCH}" == *"branch-2"* ]]; then
echo "Attempting to use run an instance on top of Hadoop 2."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-8" --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--working-dir output-integration/hadoop-2 \
--hbase-client-install "hbase-client" \
- "hbase-install" \
- "hadoop-2/bin/hadoop" \
+ hbase-install \
+ hadoop-2/bin/hadoop \
hadoop-2/share/hadoop/yarn/timelineservice \
hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-2/bin/mapred \
- >output-integration/hadoop-2.log 2>&1 ; then
+ >output-integration/hadoop-2.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
exit 2
fi
@@ -870,7 +873,12 @@ pipeline {
hbase_install_dir="hbase-hadoop3-install"
hbase_client_dir="hbase-hadoop3-client"
fi
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \
+ -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \
+ --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--working-dir output-integration/hadoop-3 \
--hbase-client-install ${hbase_client_dir} \
@@ -880,12 +888,17 @@ pipeline {
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
- >output-integration/hadoop-3.log 2>&1 ; then
+ >output-integration/hadoop-3.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
exit 2
fi
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \
+ -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \
+ --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
--working-dir output-integration/hadoop-3-shaded \
@@ -896,7 +909,8 @@ pipeline {
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
- >output-integration/hadoop-3-shaded.log 2>&1 ; then
+ >output-integration/hadoop-3-shaded.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
exit 2
fi
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 501b4833857..9292222cf52 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -509,11 +509,11 @@ public class HBaseClientReadWriteExample {
}
EOF
redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
- javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
+ $JAVA_HOME/bin/javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190.
redirect_and_run "${working_dir}/hbase-shaded-client-example" \
- java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
+ $JAVA_HOME/bin/java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
echo "Checking on results of example program."
"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 410406a1516..372c39b8f88 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -33,6 +33,11 @@ function usage {
echo " a git checkout, including ignored files."
exit 1
}
+
+MVN="mvn"
+if ! command -v mvn &>/dev/null; then
+ MVN=$MAVEN_HOME/bin/mvn
+fi
# if no args specified, show usage
if [ $# -lt 1 ]; then
usage
@@ -124,7 +129,7 @@ fi
# See http://hbase.apache.org/book.html#maven.release
echo "Maven details, in case our JDK doesn't match expectations:"
-mvn --version --offline | tee "${working_dir}/maven_version"
+${MVN} --version --offline | tee "${working_dir}/maven_version"
echo "Do a clean building of the source artifact using code in ${component_dir}"
cd "${component_dir}"
@@ -184,16 +189,16 @@ function build_tarball {
local build_log="srctarball_install.log"
local tarball_glob="hbase-*-bin.tar.gz"
if [ $build_hadoop3 -ne 0 ]; then
- local version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+ local version=$(${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:evaluate -Dexpression=project.version -q -DforceStdout)
local hadoop3_version=$(get_hadoop3_version $version)
mvn_extra_args="-Drevision=${hadoop3_version} -Dhadoop.profile=3.0"
build_log="hadoop3_srctarball_install.log"
tarball_glob="hbase-*-hadoop3-*-bin.tar.gz"
echo "Follow the ref guide section on making a RC: Step 8 Build the hadoop3 binary tarball."
else
- echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
+ echo "Follow the ref guide section on making a RC: Step 7 Build the binary tarball."
fi
- if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
+ if ${MVN} --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
assembly:single >"${working_dir}/${build_log}" 2>&1; then
for artifact in "${unpack_dir}"/hbase-assembly/target/${tarball_glob}; do
if [ -f "${artifact}" ]; then
@@ -213,20 +218,25 @@ function build_tarball {
cd "${unpack_dir}"
-build_tarball 0
+${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:active-profiles | grep -q hadoop-3.0
if [ $? -ne 0 ]; then
- exit 1
-fi
+ echo "The hadoop-3.0 profile is not activated by default, build a default tarball first."
+ # use java 8 to build with hadoop2
+ JAVA_HOME="/usr/lib/jvm/java-8" build_tarball 0
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
-mvn help:active-profiles | grep -q hadoop-3.0
-if [ $? -ne 0 ]; then
- echo "The hadoop-3.0 profile is not activated by default, build a hadoop3 tarball."
# move the previous tarballs out, so it will not be cleaned while building against hadoop3
mv "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz "${unpack_dir}"/
+ echo "build a hadoop3 tarball."
build_tarball 1
if [ $? -ne 0 ]; then
exit 1
fi
# move tarballs back
mv "${unpack_dir}"/hbase-*-bin.tar.gz "${unpack_dir}"/hbase-assembly/target/
+else
+ echo "The hadoop-3.0 profile is activated by default, build a default tarball."
+ build_tarball 0
fi