You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/02/28 03:23:19 UTC
(hbase) branch HBASE-28384-branch-2 updated (6667dce463a -> 84288604d68)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
discard 6667dce463a only run client integration test
discard 0473a89e237 HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
new 4b766e564e4 HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
new 84288604d68 only run client integration test
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6667dce463a)
\
N -- N -- N refs/heads/HBASE-28384-branch-2 (84288604d68)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/hbase_nightly_source-artifact.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
(hbase) 01/02: HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4b766e564e4e0dddc414f7d0688926136192b1d1
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Feb 26 23:11:20 2024 +0800
HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
---
dev-support/Jenkinsfile | 32 +++++++++---
dev-support/hbase_nightly_source-artifact.sh | 75 ++++++++++++++++++++++------
2 files changed, 84 insertions(+), 23 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index a767775b36b..19256ccb9b1 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -657,6 +657,8 @@ pipeline {
rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
rm -rf "hbase-install" && mkdir "hbase-install"
rm -rf "hbase-client" && mkdir "hbase-client"
+ rm -rf "hbase-hadoop3-install"
+ rm -rf "hbase-hadoop3-client"
rm -rf "hadoop-2" && mkdir "hadoop-2"
rm -rf "hadoop-3" && mkdir "hadoop-3"
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
@@ -691,14 +693,23 @@ pipeline {
"""
echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
sh '''#!/bin/bash -e
- if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then
+ if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then
echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile
exit 1
fi
- install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | head -n 1)
+ install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v client-bin | grep -v hadoop3)
tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install"
- client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1)
+ client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-client-bin.tar.gz | grep -v hadoop3)
tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client"
+ if [ 2 -eq $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | wc -l) ]; then
+ echo "hadoop3 artifacts available, unpacking the hbase hadoop3 bin tarball into 'hbase-hadoop3-install' and the client hadoop3 tarball into 'hbase-hadoop3-client'"
+ mkdir hbase-hadoop3-install
+ mkdir hbase-hadoop3-client
+ hadoop3_install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | grep -v client-bin)
+ tar --strip-component=1 -xzf "${hadoop3_install_artifact}" -C "hbase-hadoop3-install"
+ hadoop3_client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-client-bin.tar.gz)
+ tar --strip-component=1 -xzf "${hadoop3_client_artifact}" -C "hbase-hadoop3-client"
+ fi
'''
unstash 'hadoop-2'
sh '''#!/bin/bash -xe
@@ -731,11 +742,18 @@ pipeline {
tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
# we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471
${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3
+ hbase_install_dir="hbase-install"
+ hbase_client_dir="hbase-client"
+ if [ -d "hbase-hadoop3-install" ]; then
+ echo "run hadoop3 client integration test against hbase hadoop3 binaries"
+ hbase_install_dir="hbase-hadoop3-install"
+ hbase_client_dir="hbase-hadoop3-client"
+ fi
if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
--single-process \
--working-dir output-integration/hadoop-3 \
- --hbase-client-install hbase-client \
- hbase-install \
+ --hbase-client-install ${hbase_client_dir} \
+ ${hbase_install_dir} \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
@@ -750,8 +768,8 @@ pipeline {
--single-process \
--hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
--working-dir output-integration/hadoop-3-shaded \
- --hbase-client-install hbase-client \
- hbase-install \
+ --hbase-client-install ${hbase_client_dir} \
+ ${hbase_install_dir} \
hadoop-3/bin/hadoop \
hadoop-3/share/hadoop/yarn/timelineservice \
hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 5d9902e5f04..e37c90beb53 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-set -e
function usage {
echo "Usage: ${0} [options] /path/to/component/checkout"
echo ""
@@ -170,20 +169,64 @@ else
echo "Everything looks as expected."
fi
+function get_hadoop3_version {
+ local version="$1"
+ if [[ "${version}" =~ -SNAPSHOT$ ]]; then
+ echo "${version/-SNAPSHOT/-hadoop3-SNAPSHOT}"
+ else
+ echo "${version}-hadoop3"
+ fi
+}
+
+function build_tarball {
+ local build_hadoop3=$1
+ local mvn_extra_args=""
+ local build_log="srctarball_install.log"
+ local tarball_glob="hbase-*-bin.tar.gz"
+ if [ $build_hadoop3 -ne 0 ]; then
+ local version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+ local hadoop3_version=$(get_hadoop3_version $version)
+ mvn_extra_args="-Drevision=${hadoop3_version} -Dhadoop.profile=3.0"
+ build_log="hadoop3_srctarball_install.log"
+ tarball_glob="hbase-*-hadoop3-*-bin.tar.gz"
+ echo "Follow the ref guide section on making a RC: Step 8 Build the hadoop3 binary tarball."
+ else
+ echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
+ fi
+ if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} clean install \
+ assembly:single >"${working_dir}/${build_log}" 2>&1; then
+ for artifact in "${unpack_dir}"/hbase-assembly/target/${tarball_glob}; do
+ if [ -f "${artifact}" ]; then
+ # TODO check the layout of the binary artifact we just made.
+ echo "Building a binary tarball from the source tarball succeeded."
+ return 0
+ fi
+ done
+ fi
+
+ echo "Building a binary tarball from the source tarball failed. see ${working_dir}/${build_log} for details."
+ # Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
+ # rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
+ find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
+ return 1
+}
+
cd "${unpack_dir}"
-echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
-if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
- assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
- for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
- if [ -f "${artifact}" ]; then
- # TODO check the layout of the binary artifact we just made.
- echo "Building a binary tarball from the source tarball succeeded."
- exit 0
- fi
- done
+
+build_tarball 0
+if [ $? -ne 0 ]; then
+ exit 1
+fi
+
+mvn help:active-profiles | grep -q hadoop-3.0
+if [ $? -ne 0 ]; then
+ echo "The hadoop-3.0 profile is not activated by default, build a hadoop3 tarball."
+ # move the previous tarballs out, so it will not be cleaned while building against hadoop3
+ mv "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz "${unpack_dir}"/
+ build_tarball 1
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
+ # move tarballs back
+ mv "${unpack_dir}"/hbase-*-bin.tar.gz "${unpack_dir}"/hbase-assembly/target/"
fi
-echo "Building a binary tarball from the source tarball failed. see ${working_dir}/srctarball_install.log for details."
-# Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
-# rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
-find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
-exit 1
(hbase) 02/02: only run client integration test
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 84288604d68c23350800f7dbfc8bf110eb30d176
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Feb 27 18:50:32 2024 +0800
only run client integration test
---
dev-support/Jenkinsfile | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 19256ccb9b1..e099fc168d5 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -199,6 +199,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch 'branch-1*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.SHALLOW_CHECKS}"
@@ -304,7 +307,7 @@ pipeline {
}
}
when {
- branch 'branch-2*'
+ branch 'branch-1*'
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
@@ -414,6 +417,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch 'branch-1*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"
@@ -524,6 +530,9 @@ pipeline {
label 'hbase'
}
}
+ when {
+ branch 'branch-1*'
+ }
environment {
BASEDIR = "${env.WORKSPACE}/component"
TESTS = "${env.DEEP_CHECKS}"