You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/11/22 22:32:15 UTC
[1/4] hbase git commit: HBASE-19317 Set a high NodeManager max disk
utilization if not already set
Repository: hbase
Updated Branches:
refs/heads/branch-2 0ef7a2424 -> 4e387a948
refs/heads/master 548ebbc57 -> 6f0c9fbfd
HBASE-19317 Set a high NodeManager max disk utilization if not already set
This avoids the situation where the build machine has sufficient disk
space (a few GB's at most) to run an HBase test, but the default YARN
configuration would preclude the NM's from starting correctly. This
should eliminate a trivial source of build flakiness based on the host
machines being used.
Signed-off-by: Ted Yu <te...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f0c9fbf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f0c9fbf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f0c9fbf
Branch: refs/heads/master
Commit: 6f0c9fbfd1f17f5f50d90464866d153286b051a5
Parents: b0b6064
Author: Josh Elser <el...@apache.org>
Authored: Tue Nov 21 13:42:13 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 22 17:06:49 2017 -0500
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6f0c9fbf/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index bb7eca2..eec7892 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2552,6 +2552,10 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* @throws IOException When starting the cluster fails.
*/
public MiniMRCluster startMiniMapReduceCluster() throws IOException {
+ // Set a very high max-disk-utilization percentage to avoid the NodeManagers from failing.
+ conf.setIfUnset(
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage",
+ "99.0");
startMiniMapReduceCluster(2);
return mrCluster;
}
[4/4] hbase git commit: HBASE-19317 Set a high NodeManager max disk
utilization if not already set
Posted by el...@apache.org.
HBASE-19317 Set a high NodeManager max disk utilization if not already set
This avoids the situation where the build machine has sufficient disk
space (a few GB's at most) to run an HBase test, but the default YARN
configuration would preclude the NM's from starting correctly. This
should eliminate a trivial source of build flakiness based on the host
machines being used.
Signed-off-by: Ted Yu <te...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4e387a94
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4e387a94
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4e387a94
Branch: refs/heads/branch-2
Commit: 4e387a948fad9928c9d4922c9055e601d22e4145
Parents: 46cb5d5
Author: Josh Elser <el...@apache.org>
Authored: Tue Nov 21 13:42:13 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 22 17:23:53 2017 -0500
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4e387a94/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index bb7eca2..eec7892 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2552,6 +2552,10 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* @throws IOException When starting the cluster fails.
*/
public MiniMRCluster startMiniMapReduceCluster() throws IOException {
+ // Set a very high max-disk-utilization percentage to avoid the NodeManagers from failing.
+ conf.setIfUnset(
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage",
+ "99.0");
startMiniMapReduceCluster(2);
return mrCluster;
}
[2/4] hbase git commit: HBASE-19310 Avoid an NPE
IntegrationTestImportTsv when outside of the context of JUnit
Posted by el...@apache.org.
HBASE-19310 Avoid an NPE IntegrationTestImportTsv when outside of the context of JUnit
Signed-off-by: Michael Stack <st...@apache.org>
Signed-off-by: Ted Yu <te...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0b60642
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0b60642
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0b60642
Branch: refs/heads/master
Commit: b0b606429339aabe9fb964af6bf3c3129b3ac375
Parents: 548ebbc
Author: Josh Elser <el...@apache.org>
Authored: Tue Nov 21 13:34:51 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 22 17:06:49 2017 -0500
----------------------------------------------------------------------
.../hbase/mapreduce/IntegrationTestImportTsv.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0b60642/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
index 887dd8b..dfc54e0 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
@@ -185,13 +185,15 @@ public class IntegrationTestImportTsv extends Configured implements Tool {
@Test
public void testGenerateAndLoad() throws Exception {
+ generateAndLoad(TableName.valueOf(name.getMethodName()));
+ }
+
+ void generateAndLoad(final TableName table) throws Exception {
LOG.info("Running test testGenerateAndLoad.");
- final TableName table = TableName.valueOf(name.getMethodName());
String cf = "d";
Path hfiles = new Path(
util.getDataTestDirOnTestFS(table.getNameAsString()), "hfiles");
-
Map<String, String> args = new HashMap<>();
args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, hfiles.toString());
args.put(ImportTsv.COLUMNS_CONF_KEY,
@@ -226,7 +228,11 @@ public class IntegrationTestImportTsv extends Configured implements Tool {
// adding more test methods? Don't forget to add them here... or consider doing what
// IntegrationTestsDriver does.
provisionCluster();
- testGenerateAndLoad();
+ TableName tableName = TableName.valueOf("IntegrationTestImportTsv");
+ if (util.getAdmin().tableExists(tableName)) {
+ util.deleteTable(tableName);
+ }
+ generateAndLoad(tableName);
releaseCluster();
return 0;
[3/4] hbase git commit: HBASE-19310 Avoid an NPE
IntegrationTestImportTsv when outside of the context of JUnit
Posted by el...@apache.org.
HBASE-19310 Avoid an NPE IntegrationTestImportTsv when outside of the context of JUnit
Signed-off-by: Michael Stack <st...@apache.org>
Signed-off-by: Ted Yu <te...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/46cb5d59
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/46cb5d59
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/46cb5d59
Branch: refs/heads/branch-2
Commit: 46cb5d598689577b01cc7690587ae94579b70a11
Parents: 0ef7a24
Author: Josh Elser <el...@apache.org>
Authored: Tue Nov 21 13:34:51 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Nov 22 17:23:53 2017 -0500
----------------------------------------------------------------------
.../hbase/mapreduce/IntegrationTestImportTsv.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/46cb5d59/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
index 887dd8b..dfc54e0 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestImportTsv.java
@@ -185,13 +185,15 @@ public class IntegrationTestImportTsv extends Configured implements Tool {
@Test
public void testGenerateAndLoad() throws Exception {
+ generateAndLoad(TableName.valueOf(name.getMethodName()));
+ }
+
+ void generateAndLoad(final TableName table) throws Exception {
LOG.info("Running test testGenerateAndLoad.");
- final TableName table = TableName.valueOf(name.getMethodName());
String cf = "d";
Path hfiles = new Path(
util.getDataTestDirOnTestFS(table.getNameAsString()), "hfiles");
-
Map<String, String> args = new HashMap<>();
args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, hfiles.toString());
args.put(ImportTsv.COLUMNS_CONF_KEY,
@@ -226,7 +228,11 @@ public class IntegrationTestImportTsv extends Configured implements Tool {
// adding more test methods? Don't forget to add them here... or consider doing what
// IntegrationTestsDriver does.
provisionCluster();
- testGenerateAndLoad();
+ TableName tableName = TableName.valueOf("IntegrationTestImportTsv");
+ if (util.getAdmin().tableExists(tableName)) {
+ util.deleteTable(tableName);
+ }
+ generateAndLoad(tableName);
releaseCluster();
return 0;