You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2026/02/25 14:03:53 UTC
(hbase) branch HBASE-29914 updated (bae21598b21 -> 15cf8e4fd73)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-29914
in repository https://gitbox.apache.org/repos/asf/hbase.git
discard bae21598b21 fix config
omit 491271373a2 patch hadoop for hadoop2
omit 8951c661da4 skip other stages
omit d1fc9838013 HBASE-29914 Client integration tests fails with 'mvn not found'
add a3b5d7f87fb HBASE-28295 Few tests are failing due to NCDFE: org/bouncycastle/operator/OperatorCreationException (#5608)
add 18d6b00a706 [ADDENDUM] HBASE-28295 Few tests are failing due to NCDFE: org/bouncycastle/operator/OperatorCreationException (#5611)
add ab7f1feb335 HBASE-29917 TestCheckTestClasses fails with JDK8 + hadoop3 (#7787)
add 170f84f0e8c HBASE-29919 Test case TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth failed (#7794)
new 5a1e9bf3af7 HBASE-29914 Client integration tests fails with 'mvn not found'
new b6e0377ed05 skip other stages
new fe7957e2295 patch hadoop for hadoop2
new 15cf8e4fd73 fix config
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bae21598b21)
\
N -- N -- N refs/heads/HBASE-29914 (15cf8e4fd73)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hbase_nightly_pseudo-distributed-test.sh | 6 +--
hbase-it/pom.xml | 6 +++
hbase-mapreduce/pom.xml | 6 +++
hbase-server/pom.xml | 63 ++++++++++++++++++++++
.../hadoop/hbase/security/TestSecureIPC.java | 8 +--
5 files changed, 83 insertions(+), 6 deletions(-)
(hbase) 02/04: skip other stages
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-29914
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b6e0377ed0530b7c2e0b126a015e970427feecf8
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Feb 23 12:05:33 2026 +0800
skip other stages
---
dev-support/Jenkinsfile | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index eb7470eaa64..8fc8bbc98ec 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -226,6 +226,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch '*branch-3*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.SHALLOW_CHECKS}"
@@ -331,7 +334,7 @@ pipeline {
}
}
when {
- branch '*branch-2*'
+ branch '*branch-3*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
@@ -442,7 +445,7 @@ pipeline {
}
}
when {
- branch '*branch-2*'
+ branch '*branch-3*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
@@ -555,7 +558,7 @@ pipeline {
}
}
when {
- branch '*branch-2*'
+ branch '*branch-3*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
@@ -668,6 +671,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch '*branch-3*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
@@ -779,6 +785,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch '*branch-3*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
(hbase) 03/04: patch hadoop for hadoop2
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-29914
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fe7957e2295a8f70d4f0b7ea82a5b728e1f7fe8f
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Feb 24 16:30:23 2026 +0800
patch hadoop for hadoop2
---
dev-support/patch-hadoop3.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh
index fed5a535bcd..b4c51ca9487 100755
--- a/dev-support/patch-hadoop3.sh
+++ b/dev-support/patch-hadoop3.sh
@@ -20,5 +20,5 @@
hadoop_dir=$1
sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" "$hadoop_dir/libexec/hadoop-functions.sh"
-sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred"
+sed -i "/HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.MiniHadoopClusterManager/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred"
curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar"
(hbase) 01/04: HBASE-29914 Client integration tests fails with 'mvn not found'
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-29914
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5a1e9bf3af7870195a880a8ad626227883edd052
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Feb 23 12:02:35 2026 +0800
HBASE-29914 Client integration tests fails with 'mvn not found'
---
.../hbase_nightly_pseudo-distributed-test.sh | 4 +-
dev-support/hbase_nightly_source-artifact.sh | 87 +++++++++++++++++-----
dev-support/patch-hadoop3.sh | 24 ++++++
3 files changed, 96 insertions(+), 19 deletions(-)
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 923341ab43e..51905775dda 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -512,11 +512,11 @@ public class HBaseClientReadWriteExample {
}
EOF
redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
- javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
+ $JAVA_HOME/bin/javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190.
redirect_and_run "${working_dir}/hbase-shaded-client-example" \
- java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
+ $JAVA_HOME/bin/java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
echo "Checking on results of example program."
"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 5d9902e5f04..372c39b8f88 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-set -e
function usage {
echo "Usage: ${0} [options] /path/to/component/checkout"
echo ""
@@ -34,6 +33,11 @@ function usage {
echo " a git checkout, including ignored files."
exit 1
}
+
+MVN="mvn"
+if ! command -v mvn &>/dev/null; then
+ MVN=$MAVEN_HOME/bin/mvn
+fi
# if no args specified, show usage
if [ $# -lt 1 ]; then
usage
@@ -125,7 +129,7 @@ fi
# See http://hbase.apache.org/book.html#maven.release
echo "Maven details, in case our JDK doesn't match expectations:"
-mvn --version --offline | tee "${working_dir}/maven_version"
+${MVN} --version --offline | tee "${working_dir}/maven_version"
echo "Do a clean building of the source artifact using code in ${component_dir}"
cd "${component_dir}"
@@ -170,20 +174,69 @@ else
echo "Everything looks as expected."
fi
+function get_hadoop3_version {
+ local version="$1"
+ if [[ "${version}" =~ -SNAPSHOT$ ]]; then
+ echo "${version/-SNAPSHOT/-hadoop3-SNAPSHOT}"
+ else
+ echo "${version}-hadoop3"
+ fi
+}
+
+function build_tarball {
+ local build_hadoop3=$1
+ local mvn_extra_args=""
+ local build_log="srctarball_install.log"
+ local tarball_glob="hbase-*-bin.tar.gz"
+ if [ $build_hadoop3 -ne 0 ]; then
+ local version=$(${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:evaluate -Dexpression=project.version -q -DforceStdout)
+ local hadoop3_version=$(get_hadoop3_version $version)
+ mvn_extra_args="-Drevision=${hadoop3_version} -Dhadoop.profile=3.0"
+ build_log="hadoop3_srctarball_install.log"
+ tarball_glob="hbase-*-hadoop3-*-bin.tar.gz"
+ echo "Follow the ref guide section on making a RC: Step 8 Build the hadoop3 binary tarball."
+ else
+ echo "Follow the ref guide section on making a RC: Step 7 Build the binary tarball."
+ fi
+ if ${MVN} --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
+ assembly:single >"${working_dir}/${build_log}" 2>&1; then
+ for artifact in "${unpack_dir}"/hbase-assembly/target/${tarball_glob}; do
+ if [ -f "${artifact}" ]; then
+ # TODO check the layout of the binary artifact we just made.
+ echo "Building a binary tarball from the source tarball succeeded."
+ return 0
+ fi
+ done
+ fi
+
+ echo "Building a binary tarball from the source tarball failed. see ${working_dir}/${build_log} for details."
+ # Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
+ # rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
+ find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
+ return 1
+}
+
cd "${unpack_dir}"
-echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
-if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
- assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
- for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
- if [ -f "${artifact}" ]; then
- # TODO check the layout of the binary artifact we just made.
- echo "Building a binary tarball from the source tarball succeeded."
- exit 0
- fi
- done
+
+${MVN} -Dmaven.repo.local="${m2_tarbuild}" help:active-profiles | grep -q hadoop-3.0
+if [ $? -ne 0 ]; then
+ echo "The hadoop-3.0 profile is not activated by default, build a default tarball first."
+ # use java 8 to build with hadoop2
+ JAVA_HOME="/usr/lib/jvm/java-8" build_tarball 0
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
+
+ # move the previous tarballs out, so it will not be cleaned while building against hadoop3
+ mv "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz "${unpack_dir}"/
+ echo "build a hadoop3 tarball."
+ build_tarball 1
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
+ # move tarballs back
+ mv "${unpack_dir}"/hbase-*-bin.tar.gz "${unpack_dir}"/hbase-assembly/target/
+else
+ echo "The hadoop-3.0 profile is activated by default, build a default tarball."
+ build_tarball 0
fi
-echo "Building a binary tarball from the source tarball failed. see ${working_dir}/srctarball_install.log for details."
-# Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
-# rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
-find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
-exit 1
diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh
new file mode 100755
index 00000000000..fed5a535bcd
--- /dev/null
+++ b/dev-support/patch-hadoop3.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+hadoop_dir=$1
+
+sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" "$hadoop_dir/libexec/hadoop-functions.sh"
+sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred"
+curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar"
(hbase) 04/04: fix config
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-29914
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 15cf8e4fd73d0c97289c225f4dc07078682f5a72
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Feb 25 22:03:35 2026 +0800
fix config
---
dev-support/hbase_nightly_pseudo-distributed-test.sh | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 51905775dda..55f14efd3b6 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -322,6 +322,14 @@ echo "Listing HDFS contents"
redirect_and_run "${working_dir}/hadoop_cluster_smoke" \
"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ # for now, all hbase branches are compiled with hadoop 3.4.x when using hadoop-3.0 profile, where
+ # the protobuf library has been shaded and relocated, so we always need to use ProtobufRpcEngine2
+ # at hbase side, even if the hadoop server side uses ProtobufRpcEngine, so here we will remove the
+ # config value to let the hadoop code pick the right rpc engine
+ sed -i "/<property>.*ProtobufRpcEngine.*<\/property>/d" "${working_dir}/hbase-conf/core-site.xml"
+fi
+
echo "Starting up HBase"
HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh"