You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/02/27 15:02:43 UTC

(hbase) branch HBASE-28384-branch-2 updated (31170d75c7f -> 39d99fb8eb7)

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a change to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


 discard 31170d75c7f only run client integration test
 discard 5f0f26d1238 HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
     new ad0dce2c37b HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
     new 39d99fb8eb7 only run client integration test

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (31170d75c7f)
            \
             N -- N -- N   refs/heads/HBASE-28384-branch-2 (39d99fb8eb7)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 dev-support/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)


(hbase) 02/02: only run client integration test

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit 39d99fb8eb7bcb5c4babe3ea17b1ceefbb7d05af
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Feb 27 18:50:32 2024 +0800

    only run client integration test
---
 dev-support/Jenkinsfile | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 19256ccb9b1..e099fc168d5 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -199,6 +199,9 @@ pipeline {
               label 'hbase'
             }
           }
+          when {
+            branch 'branch-1*'
+          }
           environment {
             BASEDIR = "${env.WORKSPACE}/component"
             TESTS = "${env.SHALLOW_CHECKS}"
@@ -304,7 +307,7 @@ pipeline {
             }
           }
           when {
-            branch 'branch-2*'
+            branch 'branch-1*'
           }
           environment {
             BASEDIR = "${env.WORKSPACE}/component"
@@ -414,6 +417,9 @@ pipeline {
               label 'hbase'
             }
           }
+          when {
+            branch 'branch-1*'
+          }
           environment {
             BASEDIR = "${env.WORKSPACE}/component"
             TESTS = "${env.DEEP_CHECKS}"
@@ -524,6 +530,9 @@ pipeline {
               label 'hbase'
             }
           }
+          when {
+            branch 'branch-1*'
+          }
           environment {
             BASEDIR = "${env.WORKSPACE}/component"
             TESTS = "${env.DEEP_CHECKS}"


(hbase) 01/02: HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch HBASE-28384-branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git

commit ad0dce2c37b8ed4ceda33b5d4eb9841d9d36363a
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Feb 26 23:11:20 2024 +0800

    HBASE-28384 Client ingegration tests fails for branch-2/branch-2.6
---
 dev-support/Jenkinsfile                      | 32 ++++++++++---
 dev-support/hbase_nightly_source-artifact.sh | 72 +++++++++++++++++++++-------
 2 files changed, 81 insertions(+), 23 deletions(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index a767775b36b..19256ccb9b1 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -657,6 +657,8 @@ pipeline {
               rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
               rm -rf "hbase-install" && mkdir "hbase-install"
               rm -rf "hbase-client" && mkdir "hbase-client"
+              rm -rf "hbase-hadoop3-install"
+              rm -rf "hbase-hadoop3-client"
               rm -rf "hadoop-2" && mkdir "hadoop-2"
               rm -rf "hadoop-3" && mkdir "hadoop-3"
               rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
@@ -691,14 +693,23 @@ pipeline {
             """
             echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
             sh '''#!/bin/bash -e
-              if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then
+              if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v hadoop3 | wc -l) ]; then
                 echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile
                 exit 1
               fi
-              install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | head -n 1)
+              install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | grep -v client-bin | grep -v hadoop3)
               tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install"
-              client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1)
+              client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-client-bin.tar.gz | grep -v hadoop3)
               tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client"
+              if [ 2 -eq $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | wc -l) ]; then
+                echo "hadoop3 artifacts available, unpacking the hbase hadoop3 bin tarball into 'hbase-hadoop3-install' and the client hadoop3 tarball into 'hbase-hadoop3-client'"
+                mkdir hbase-hadoop3-install
+                mkdir hbase-hadoop3-client
+                hadoop3_install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-bin.tar.gz | grep -v client-bin)
+                tar --strip-component=1 -xzf "${hadoop3_install_artifact}" -C "hbase-hadoop3-install"
+                hadoop3_client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-hadoop3-*-client-bin.tar.gz)
+                tar --strip-component=1 -xzf "${hadoop3_client_artifact}" -C "hbase-hadoop3-client"
+              fi
             '''
             unstash 'hadoop-2'
             sh '''#!/bin/bash -xe
@@ -731,11 +742,18 @@ pipeline {
               tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
               # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471
               ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3
+              hbase_install_dir="hbase-install"
+              hbase_client_dir="hbase-client"
+              if [ -d "hbase-hadoop3-install" ]; then
+                echo "run hadoop3 client integration test against hbase hadoop3 binaries"
+                hbase_install_dir="hbase-hadoop3-install"
+                hbase_client_dir="hbase-hadoop3-client"
+              fi
               if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
                   --single-process \
                   --working-dir output-integration/hadoop-3 \
-                  --hbase-client-install hbase-client \
-                  hbase-install \
+                  --hbase-client-install ${hbase_client_dir} \
+                  ${hbase_install_dir} \
                   hadoop-3/bin/hadoop \
                   hadoop-3/share/hadoop/yarn/timelineservice \
                   hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
@@ -750,8 +768,8 @@ pipeline {
                   --single-process \
                   --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
                   --working-dir output-integration/hadoop-3-shaded \
-                  --hbase-client-install hbase-client \
-                  hbase-install \
+                  --hbase-client-install ${hbase_client_dir} \
+                  ${hbase_install_dir} \
                   hadoop-3/bin/hadoop \
                   hadoop-3/share/hadoop/yarn/timelineservice \
                   hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 5d9902e5f04..2173bb71ffb 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -16,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
 function usage {
   echo "Usage: ${0} [options] /path/to/component/checkout"
   echo ""
@@ -170,20 +169,61 @@ else
   echo "Everything looks as expected."
 fi
 
+function get_hadoop3_version {
+  local version="$1"
+  if [[ "${version}" =~ -SNAPSHOT$ ]]; then
+    echo "${version/-SNAPSHOT/-hadoop3-SNAPSHOT}"
+  else
+    echo "${version}-hadoop3"
+  fi
+}
+
+function build_tarball {
+  local build_hadoop3=$1
+  local mvn_extra_args=""
+  local build_log="srctarball_install.log"
+  local tarball_glob="hbase-*-bin.tar.gz"
+  if [ $build_hadoop3 -ne 0 ]; then
+    local version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+    local hadoop3_version=$(get_hadoop3_version $version)
+    mvn_extra_args="-Drevision=${hadoop3_version} -Dhadoop.profile=3.0"
+    build_log="hadoop3_srctarball_install.log"
+    tarball_glob="hbase-*-hadoop3-*-bin.tar.gz"
+    echo "Follow the ref guide section on making a RC: Step 8 Build the hadoop3 binary tarball."
+  else
+    echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
+  fi
+  if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" ${mvn_extra_args} install \
+    assembly:single >"${working_dir}/${build_log}" 2>&1; then
+    for artifact in "${unpack_dir}"/hbase-assembly/target/${tarball_glob}; do
+      if [ -f "${artifact}" ]; then
+        # TODO check the layout of the binary artifact we just made.
+        echo "Building a binary tarball from the source tarball succeeded."
+        return 0
+      fi
+    done
+  fi
+
+  echo "Building a binary tarball from the source tarball failed. see ${working_dir}/${build_log} for details."
+  # Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
+  # rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
+  find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
+  return 1
+}
+
 cd "${unpack_dir}"
-echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
-if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
-    assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
-  for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
-    if [ -f "${artifact}" ]; then
-      # TODO check the layout of the binary artifact we just made.
-      echo "Building a binary tarball from the source tarball succeeded."
-      exit 0
-    fi
-  done
+mvn clean
+
+build_tarball 0
+if [ $? -ne 0 ]; then
+  exit 1
+fi
+
+mvn help:active-profiles | grep -q hadoop-3.0
+if [ $? -ne 0 ]; then
+  echo "The hadoop-3.0 profile is not activated by default, build a hadoop3 tarball."
+  build_tarball 1
+  if [ $? -ne 0 ]; then
+    exit 1
+  fi
 fi
-echo "Building a binary tarball from the source tarball failed. see ${working_dir}/srctarball_install.log for details."
-# Copy up the rat.txt to the working dir so available in build archive in case rat complaints.
-# rat.txt can be under any module target dir... copy them all up renaming them to include parent dir as we go.
-find ${unpack_dir} -name rat.txt -type f | while IFS= read -r NAME; do cp -v "$NAME" "${working_dir}/${NAME//\//_}"; done
-exit 1