You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/06/29 13:42:18 UTC
(hbase) branch branch-3 updated (3160b36ffd7 -> 52babd94263)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
from 3160b36ffd7 HBASE-28687 BackupSystemTable#checkSystemTable should ensure system tables are enabled (#6018)
new 616287367f2 HBASE-28694 Make client integration and packaging test work with java 17 (#6035)
new 52babd94263 HBASE-28699 Bump jdk and maven versions in pre commit and nighly dockerfile (#6034)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 50 ++++++++++++++--------
dev-support/docker/Dockerfile | 16 +++----
.../hbase_nightly_pseudo-distributed-test.sh | 4 +-
dev-support/hbase_nightly_source-artifact.sh | 30 ++++++++-----
4 files changed, 62 insertions(+), 38 deletions(-)
(hbase) 01/02: HBASE-28694 Make client integration and packaging test work with java 17 (#6035)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 616287367f259ac3d705be428a9143d3039bf44a
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Jun 29 21:36:57 2024 +0800
HBASE-28694 Make client integration and packaging test work with java 17 (#6035)
Signed-off-by: Xin Sun <su...@apache.org>
(cherry picked from commit c722dde59fcfc8b5e176e8e92923d759bd761163)
---
dev-support/Jenkinsfile | 50 ++++++++++++++--------
.../hbase_nightly_pseudo-distributed-test.sh | 4 +-
dev-support/hbase_nightly_source-artifact.sh | 30 ++++++++-----
3 files changed, 54 insertions(+), 30 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 0a47726e53c..94652560614 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -758,11 +758,6 @@ pipeline {
label 'hbase-large'
}
}
- tools {
- maven 'maven_latest'
- // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
- jdk "jdk_1.8_latest"
- }
environment {
BASEDIR = "${env.WORKSPACE}/component"
BRANCH = "${env.BRANCH_NAME}"
@@ -797,21 +792,25 @@ pipeline {
echo "got the following saved stats in 'output-srctarball/machine'"
ls -lh "output-srctarball/machine"
'''
- sh """#!/bin/bash -e
+ sh '''#!/bin/bash -e
echo "Checking the steps for an RM to make a source artifact, then a binary artifact."
- if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" --workdir=/hbase hbase-integration-test \
+ "component/dev-support/hbase_nightly_source-artifact.sh" \
--intermediate-file-dir output-srctarball \
--unpack-temp-dir unpacked_src_tarball \
--maven-m2-initial .m2-for-repo \
--maven-m2-src-build .m2-for-src \
--clean-source-checkout \
- "${env.BASEDIR}" ; then
+ component
+ if [ $? -eq 0 ]; then
echo '(/) {color:green}+1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
else
echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
exit 1
fi
- """
+ '''
echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
sh '''#!/bin/bash -e
if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then
@@ -834,21 +833,25 @@ pipeline {
'''
unstash 'hadoop-2'
sh '''#!/bin/bash -xe
- if [[ "${BRANCH}" = branch-2* ]]; then
+ if [[ "${BRANCH}" == *"branch-2"* ]]; then
echo "Attempting to use run an instance on top of Hadoop 2."
artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-8" --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--working-dir output-integration/hadoop-2 \
--hbase-client-install "hbase-client" \
- "hbase-install" \
- "hadoop-2/bin/hadoop" \
+ hbase-install \
+ hadoop-2/bin/hadoop \
hadoop-2/share/hadoop/yarn/timelineservice \
hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-2/bin/mapred \
- >output-integration/hadoop-2.log 2>&1 ; then
+ >output-integration/hadoop-2.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
exit 2
fi
@@ -870,7 +873,12 @@ pipeline {
hbase_install_dir="hbase-hadoop3-install"
hbase_client_dir="hbase-hadoop3-client"
fi
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" .
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \
+ -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \
+ --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--working-dir output-integration/hadoop-3 \
--hbase-client-install ${hbase_client_dir} \
@@ -880,12 +888,17 @@ pipeline {
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
- >output-integration/hadoop-3.log 2>&1 ; then
+ >output-integration/hadoop-3.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
exit 2
fi
echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \
+ -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \
+ -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \
+ --workdir=/hbase hbase-integration-test \
+ component/dev-support/hbase_nightly_pseudo-distributed-test.sh \
--single-process \
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
--working-dir output-integration/hadoop-3-shaded \
@@ -896,7 +909,8 @@ pipeline {
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
hadoop-3/bin/mapred \
- >output-integration/hadoop-3-shaded.log 2>&1 ; then
+ >output-integration/hadoop-3-shaded.log 2>&1
+ if [ $? -ne 0 ]; then
echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
exit 2
fi
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 8290f06110f..56fec01aef7 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -513,11 +513,11 @@ public class HBaseClientReadWriteExample {
}
EOF
redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
- javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
+ $JAVA_HOME/bin/javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190.
redirect_and_run "${working_dir}/hbase-shaded-client-example" \
- java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
+ $JAVA_HOME/bin/java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
echo "Checking on results of example program."
"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 79c62ca7743..59667408cfa 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -33,6 +33,11 @@ function usage {
echo " a git checkout, including ignored files."
exit 1
}
+
+MVN="mvn"
+if ! command -v mvn &>/dev/null; then
+ MVN=$MAVEN_HOME/bin/mvn
+fi
# if no args specified, show usage
if [ $# -lt 1 ]; then
usage
@@ -124,7 +129,7 @@ fi
# See http://hbase.apache.org/book.html#maven.release
echo "Maven details, in case our JDK doesn't match expectations:"
-mvn --version --offline | tee "${working_dir}/maven_version"
+${MVN} --version --offline | tee "${working_dir}/maven_version"
echo "Do a clean building of the source artifact using code in ${component_dir}"
cd "${component_dir}"
@@ -183,16 +188,16 @@ function build_tarball {
local build_log="srctarball_install.log"
local tarball_glob="hbase-*-bin.tar.gz"
if [ $build_hadoop3 -ne 0 ]; then
- local version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+ local version=$(${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:evaluate -Dexpression=project.version -q -DforceStdout)
local hadoop3_version=$(get_hadoop3_version $version)
mvn_extra_args="-Drevision=${hadoop3_version} -Dhadoop.profile=3.0"
build_log="hadoop3_srctarball_install.log"
tarball_glob="hbase-*-hadoop3-*-bin.tar.gz"
echo "Follow the ref guide section on making a RC: Step 8 Build the hadoop3 binary tarball."
else
- echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
+ echo "Follow the ref guide section on making a RC: Step 7 Build the binary tarball."
fi
- if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
+ if ${MVN} --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
assembly:single >"${working_dir}/${build_log}" 2>&1; then
for artifact in "${unpack_dir}"/hbase-assembly/target/${tarball_glob}; do
if [ -f "${artifact}" ]; then
@@ -212,20 +217,25 @@ function build_tarball {
cd "${unpack_dir}"
-build_tarball 0
+${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:active-profiles | grep -q hadoop-3.0
if [ $? -ne 0 ]; then
- exit 1
-fi
+ echo "The hadoop-3.0 profile is not activated by default, build a default tarball first."
+ # use java 8 to build with hadoop2
+ JAVA_HOME="/usr/lib/jvm/java-8" build_tarball 0
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
-mvn help:active-profiles | grep -q hadoop-3.0
-if [ $? -ne 0 ]; then
- echo "The hadoop-3.0 profile is not activated by default, build a hadoop3 tarball."
# move the previous tarballs out, so it will not be cleaned while building against hadoop3
mv "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz "${unpack_dir}"/
+ echo "build a hadoop3 tarball."
build_tarball 1
if [ $? -ne 0 ]; then
exit 1
fi
# move tarballs back
mv "${unpack_dir}"/hbase-*-bin.tar.gz "${unpack_dir}"/hbase-assembly/target/
+else
+ echo "The hadoop-3.0 profile is activated by default, build a default tarball."
+ build_tarball 0
fi
(hbase) 02/02: HBASE-28699 Bump jdk and maven versions in pre commit and nighly dockerfile (#6034)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 52babd942632ab5603b3f545269e3450ee9a9c52
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Jun 29 21:37:27 2024 +0800
HBASE-28699 Bump jdk and maven versions in pre commit and nighly dockerfile (#6034)
Signed-off-by: Xin Sun <su...@apache.org>
(cherry picked from commit e6395a0dea10c1773b32d61afab4a54c64e05d2b)
---
dev-support/docker/Dockerfile | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 06e4b67526f..26b2c35b346 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -91,30 +91,30 @@ RUN curl --location --fail --silent --show-error --output /tmp/hadolint "${HADOL
echo "${HADOLINT_SHA512} */tmp/hadolint" | sha512sum -c -
FROM base_image AS maven_download_image
-ENV MAVEN_VERSION='3.8.6'
+ENV MAVEN_VERSION='3.9.8'
ENV MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
-ENV MAVEN_SHA512='f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26'
+ENV MAVEN_SHA512='7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/maven.tar.gz "${MAVEN_URL}" && \
echo "${MAVEN_SHA512} */tmp/maven.tar.gz" | sha512sum -c -
FROM base_image AS openjdk8_download_image
-ENV OPENJDK8_URL='https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u352-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u352b08.tar.gz'
-ENV OPENJDK8_SHA256='1633bd7590cb1cd72f5a1378ae8294451028b274d798e2a4ac672059a2f00fee'
+ENV OPENJDK8_URL='https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u412-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u412b08.tar.gz'
+ENV OPENJDK8_SHA256='b9884a96f78543276a6399c3eb8c2fd8a80e6b432ea50e87d3d12d495d1d2808'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk8.tar.gz "${OPENJDK8_URL}" && \
echo "${OPENJDK8_SHA256} */tmp/adoptopenjdk8.tar.gz" | sha256sum -c -
FROM base_image AS openjdk11_download_image
-ENV OPENJDK11_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.17_8.tar.gz'
-ENV OPENJDK11_SHA256='b8d46ed08ef4859476fe6421a7690d899ed83dce63f13fd894f994043177ef3c'
+ENV OPENJDK11_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_x64_linux_hotspot_11.0.23_9.tar.gz'
+ENV OPENJDK11_SHA256='23e47ea7a3015be3240f21185fd902adebdcf76530757c9b482c7eb5bd3417c2'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -
FROM base_image AS openjdk17_download_image
-ENV OPENJDK17_URL='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
-ENV OPENJDK17_SHA256='a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
+ENV OPENJDK17_URL='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz'
+ENV OPENJDK17_SHA256='aa7fb6bb342319d227a838af5c363bfa1b4a670c209372f9e6585bd79da6220c'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -