[hbase] branch branch-2 updated: HBASE-26828 Increase the concurrency when running UTs in pre commit job (#4213)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2 by this push:
     new 2a5b340  HBASE-26828 Increase the concurrency when running UTs in pre commit job (#4213)
2a5b340 is described below

commit 2a5b3407b16bdd16cea4d5df4bd5d032e7a0a4c2
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Mar 14 12:01:51 2022 +0800

    HBASE-26828 Increase the concurrency when running UTs in pre commit job (#4213)
    
    Signed-off-by: Yulin Niu <ni...@apache.org>
    (cherry picked from commit a49d147d49b079c3974e9a29fd1a6d861ca6b525)
---
 dev-support/Jenkinsfile_GitHub                |  5 ++++-
 dev-support/hbase-personality.sh              | 29 ++++++++++++++++++++++++++-
 dev-support/jenkins_precommit_github_yetus.sh | 10 +++++++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index 12a83ce..4eee610 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -45,7 +45,10 @@ pipeline {
         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
-
+        // set build parallel
+        BUILD_THREAD = 4
+        SUREFIRE_FIRST_PART_FORK_COUNT = '0.5C'
+        SUREFIRE_SECOND_PART_FORK_COUNT = '0.5C'
         // a global view of paths. parallel stages can land on the same host concurrently, so each
         // stage works in its own subdirectory. there is an "output" under each of these
         // directories, which we retrieve after the build is complete.
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index cccaaac..fd3b09f 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -119,6 +119,18 @@ function personality_parse_args
         delete_parameter "${i}"
         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
       ;;
+      --build-thread=*
+        delete_parameter "${i}"
+        BUILD_THREAD=${i#*=}
+      ;;
+      --surefire-first-part-fork-count=*
+        delete_parameter "${i}"
+        SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
+      ;;
+      --surefire-second-part-fork-count=*
+        delete_parameter "${i}"
+        SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
+      ;;
     esac
   done
 }
@@ -144,7 +156,13 @@ function personality_modules
   # At a few points, hbase modules can run build, test, etc. in parallel
   # Let it happen. Means we'll use more CPU but should be for short bursts.
   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
-  extra="--threads=2 -DHBasePatchProcess"
+  if [[ -n "${BUILD_THREAD}" ]]; then
+    extra="--threads=${BUILD_THREAD}"
+  else
+    extra="--threads=2"
+  fi
+
+  extra="${extra} -DHBasePatchProcess"
   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
     extra="${extra} -Dhttps.protocols=TLSv1.2"
   fi
@@ -232,6 +250,15 @@ function personality_modules
       extra="${extra} -Dbuild.id=${BUILD_ID}"
     fi
 
+    # set forkCount
+    if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
+      extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
+    fi
+
+    if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
+      extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
+    fi
+
     # If the set of changed files includes CommonFSUtils then add the hbase-server
     # module to the set of modules (if not already included) to be tested
     for f in "${CHANGED_FILES[@]}"
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index 693e07b..c3da326 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -147,6 +147,16 @@ YETUS_ARGS+=("--github-use-emoji-vote")
 if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
   YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
 fi
+# pass build parallelism in
+if [[ -n "${BUILD_THREAD}" ]]; then
+  YETUS_ARGS+=("--build-thread=${BUILD_THREAD}")
+fi
+if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
+  YETUS_ARGS+=("--surefire-first-part-fork-count=${SUREFIRE_FIRST_PART_FORK_COUNT}")
+fi
+if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
+  YETUS_ARGS+=("--surefire-second-part-fork-count=${SUREFIRE_SECOND_PART_FORK_COUNT}")
+fi
 
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"