You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/04/13 18:44:59 UTC
[hbase] branch branch-2.3 updated (f6ab525 -> 117164e)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a change to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from f6ab525 HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundException Overwrite yarn configs that write /tmp dir by default.
new 0931019 Revert "HBASE-24126 Up the container nproc uplimit from 10000 to 12500 (#1450)"
new 117164e HBASE-24167 [Flakey Tests] TestHRegionWithInMemoryFlush#testWritesWhileScanning gets stuck MVCC
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/flaky-tests/flaky-reporting.Jenkinsfile | 2 +-
dev-support/hbase-personality.sh | 5 +----
dev-support/hbase_docker.sh | 2 +-
dev-support/hbase_nightly_yetus.sh | 2 +-
dev-support/jenkins_precommit_github_yetus.sh | 2 +-
.../apache/hadoop/hbase/regionserver/TestHRegion.java | 18 ++++++++++++++++--
.../regionserver/TestHRegionWithInMemoryFlush.java | 12 ++++++++++++
7 files changed, 33 insertions(+), 10 deletions(-)
[hbase] 01/02: Revert "HBASE-24126 Up the container nproc uplimit
from 10000 to 12500 (#1450)"
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0931019681ddeae29db196c67241813f13bc9c92
Author: stack <st...@apache.org>
AuthorDate: Fri Apr 10 16:04:34 2020 -0700
Revert "HBASE-24126 Up the container nproc uplimit from 10000 to 12500 (#1450)"
This reverts commit f5b3945ea4e0fdd5f8fa521db34fd39fc74ec3ba.
---
dev-support/flaky-tests/flaky-reporting.Jenkinsfile | 2 +-
dev-support/hbase-personality.sh | 5 +----
dev-support/hbase_docker.sh | 2 +-
dev-support/hbase_nightly_yetus.sh | 2 +-
dev-support/jenkins_precommit_github_yetus.sh | 2 +-
5 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
index 58e7328..37d557c 100644
--- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
+++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
@@ -43,7 +43,7 @@ pipeline {
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 10)
flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 30)
docker build -t hbase-dev-support dev-support
- docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
+ docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
'''
}
}
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 68bfda3..ec05187 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -81,11 +81,8 @@ function personality_globals
# Yetus 0.7.0 enforces limits. Default proclimit is 1000.
# Up it. See HBASE-19902 for how we arrived at this number.
- # NOTE: I don't think changing this has an effect. Set the
- # --proclimit passed to yetus. This seems to do what we
- # need changing proclimit.
#shellcheck disable=SC2034
- PROCLIMIT=12500
+ PROCLIMIT=10000
# Set docker container to run with 20g. Default is 4g in yetus.
# See HBASE-19902 for how we arrived at 20g.
diff --git a/dev-support/hbase_docker.sh b/dev-support/hbase_docker.sh
index 59a15f7..cc17cf7 100755
--- a/dev-support/hbase_docker.sh
+++ b/dev-support/hbase_docker.sh
@@ -159,4 +159,4 @@ done
echo "Successfully built ${IMAGE_NAME}."
echo "Starting hbase shell..."
-docker run --ulimit nproc=12500 -it ${IMAGE_NAME}
+docker run -it ${IMAGE_NAME}
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 32724ba..11a44e0 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -66,7 +66,7 @@ YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--ignore-unknown-options=true" "${YETUS_ARGS[@]}")
# Why are these not being picked up from hbase-personality?
-YETUS_ARGS=("--proclimit=12500" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index a1d5720..e0ead27 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -102,7 +102,7 @@ YETUS_ARGS+=("--reapermode=kill")
# set relatively high limits for ASF machines
# changing these to higher values may cause problems
# with other jobs on systemd-enabled machines
-YETUS_ARGS+=("--proclimit=12500")
+YETUS_ARGS+=("--proclimit=10000")
YETUS_ARGS+=("--dockermemlimit=20g")
# -1 spotbugs issues that show up prior to the patch being applied
YETUS_ARGS+=("--spotbugs-strict-precheck")
[hbase] 02/02: HBASE-24167 [Flakey Tests]
TestHRegionWithInMemoryFlush#testWritesWhileScanning gets stuck MVCC
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 117164e12f5eca4536039da66a5997df0bb527e5
Author: stack <st...@apache.org>
AuthorDate: Fri Apr 10 18:30:08 2020 -0700
HBASE-24167 [Flakey Tests] TestHRegionWithInMemoryFlush#testWritesWhileScanning gets stuck MVCC
---
.../apache/hadoop/hbase/regionserver/TestHRegion.java | 18 ++++++++++++++++--
.../regionserver/TestHRegionWithInMemoryFlush.java | 12 ++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 09a60a1..5c6f117 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3816,6 +3816,20 @@ public class TestHRegion {
}
/**
+ * So can be overridden in subclasses.
+ */
+ int getNumQualifiersForTestWritesWhileScanning() {
+ return 100;
+ }
+
+ /**
+ * So can be overridden in subclasses.
+ */
+ int getTestCountForTestWritesWhileScanning() {
+ return 100;
+ }
+
+ /**
* Writes very wide records and scans for the latest every time.. Flushes and
* compacts the region every now and then to keep things realistic.
*
@@ -3826,10 +3840,10 @@ public class TestHRegion {
*/
@Test
public void testWritesWhileScanning() throws IOException, InterruptedException {
- int testCount = 100;
+ int testCount = getTestCountForTestWritesWhileScanning();
int numRows = 1;
int numFamilies = 10;
- int numQualifiers = 100;
+ int numQualifiers = getNumQualifiersForTestWritesWhileScanning();
int flushInterval = 7;
int compactInterval = 5 * flushInterval;
byte[][] families = new byte[numFamilies][];
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
index 3a64648..c539443 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
@@ -68,6 +68,18 @@ public class TestHRegionWithInMemoryFlush extends TestHRegion {
isReadOnly, durability, wal, inMemory, families);
}
+ @Override int getTestCountForTestWritesWhileScanning() {
+ return 10;
+ }
+
+ /**
+ * testWritesWhileScanning is flakey when called out of this class. Need to dig in. Meantime
+ * go easy on it. See if that helps.
+ */
+ @Override int getNumQualifiersForTestWritesWhileScanning() {
+ return 10;
+ }
+
/**
* A test case of HBASE-21041
* @throws Exception Exception