You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zg...@apache.org on 2020/08/17 02:08:37 UTC
[hbase] branch branch-2.2 updated (9e69503 -> 8a6dcc3)
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a change to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 9e69503 HBASE-24881 Fix flaky TestMasterAbortAndRSGotKilled for branch-2.2 (#2258)
new 4d689f0 HBASE-24150 Allow module tests run in parallel (#1464)
new 8a6dcc3 HBASE-24126 Up the container nproc uplimit from 10000 to 12500 (#1504)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/flaky-tests/flaky-reporting.Jenkinsfile | 2 +-
dev-support/hbase-personality.sh | 7 +++++--
dev-support/hbase_docker.sh | 2 +-
dev-support/hbase_nightly_source-artifact.sh | 2 +-
dev-support/hbase_nightly_yetus.sh | 2 --
pom.xml | 4 ++--
6 files changed, 10 insertions(+), 9 deletions(-)
[hbase] 02/02: HBASE-24126 Up the container nproc uplimit from
10000 to 12500 (#1504)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8a6dcc3ae0234a8e914966fb856411f5683ed882
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Mon Apr 13 14:59:16 2020 -0700
HBASE-24126 Up the container nproc uplimit from 10000 to 12500 (#1504)
---
dev-support/flaky-tests/flaky-reporting.Jenkinsfile | 2 +-
dev-support/hbase-personality.sh | 2 +-
dev-support/hbase_docker.sh | 2 +-
dev-support/hbase_nightly_yetus.sh | 2 --
4 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
index c42bc64..8a56c0b 100644
--- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
+++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
@@ -43,7 +43,7 @@ pipeline {
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 10)
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 30)
docker build -t hbase-dev-support dev-support
- docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
+ docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
'''
}
}
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index d87eea6..0b9062e 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -85,7 +85,7 @@ function personality_globals
# Yetus 0.7.0 enforces limits. Default proclimit is 1000.
# Up it. See HBASE-19902 for how we arrived at this number.
#shellcheck disable=SC2034
- PROCLIMIT=10000
+ PROC_LIMIT=12500
# Set docker container to run with 20g. Default is 4g in yetus.
# See HBASE-19902 for how we arrived at 20g.
diff --git a/dev-support/hbase_docker.sh b/dev-support/hbase_docker.sh
index cc17cf7..59a15f7 100755
--- a/dev-support/hbase_docker.sh
+++ b/dev-support/hbase_docker.sh
@@ -159,4 +159,4 @@ done
echo "Successfully built ${IMAGE_NAME}."
echo "Starting hbase shell..."
-docker run -it ${IMAGE_NAME}
+docker run --ulimit nproc=12500 -it ${IMAGE_NAME}
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 564b849..6c4df2c 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -69,8 +69,6 @@ YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--ignore-unknown-options=true" "${YETUS_ARGS[@]}")
-# Why are these not being picked up from hbase-personality?
-YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
[hbase] 01/02: HBASE-24150 Allow module tests run in parallel
(#1464)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4d689f0a72fddc425219ea396e27172d699a350c
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Thu Apr 9 14:41:41 2020 -0700
HBASE-24150 Allow module tests run in parallel (#1464)
Pass --threads=2 to mvn when yetus runs so some parallelism
when dependencies allow.
Signed-off-by: Nick Dimiduk <nd...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
dev-support/hbase-personality.sh | 5 ++++-
dev-support/hbase_nightly_source-artifact.sh | 2 +-
pom.xml | 4 ++--
3 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index d0e0f2d..d87eea6 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -140,7 +140,10 @@ function personality_modules
clear_personality_queue
- extra="-DHBasePatchProcess"
+ # At a few points, hbase modules can run build, test, etc. in parallel
+ # Let it happen. Means we'll use more CPU but should be for short bursts.
+ # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
+ extra="--threads=2 -DHBasePatchProcess"
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
extra="${extra} -Dhttps.protocols=TLSv1.2"
fi
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index cd17a32..87c3f3f 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -172,7 +172,7 @@ fi
cd "${unpack_dir}"
echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
-if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
+if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
if [ -f "${artifact}" ]; then
diff --git a/pom.xml b/pom.xml
index cd21e05..c585d6f 100755
--- a/pom.xml
+++ b/pom.xml
@@ -3852,8 +3852,8 @@
<id>hbase.apache.org</id>
<name>HBase Website at hbase.apache.org</name>
<!-- On why this is the tmp dir and not hbase.apache.org, see
- https://issues.apache.org/jira/browse/HBASE-7593?focusedCommentId=13555866&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13555866
- -->
+ https://issues.apache.org/jira/browse/HBASE-7593?focusedCommentId=13555866&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13555866
+ -->
<url>file:///tmp</url>
</site>
</distributionManagement>