You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bi...@apache.org on 2019/12/02 02:34:25 UTC
[hbase] branch branch-2.2 updated: HBASE-23352: Allow chaos monkeys
to access cmd line params, and improve FillDiskCommandAction (#885)
This is an automated email from the ASF dual-hosted git repository.
binlijin pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new 4b176a5 HBASE-23352: Allow chaos monkeys to access cmd line params, and improve FillDiskCommandAction (#885)
4b176a5 is described below
commit 4b176a5f3ba63507c1fd694228c25f35f4a61b0a
Author: BukrosSzabolcs <bu...@gmail.com>
AuthorDate: Mon Dec 2 03:29:06 2019 +0100
HBASE-23352: Allow chaos monkeys to access cmd line params, and improve FillDiskCommandAction (#885)
Instead of using the default properties when checking for monkey
properties, now we use the ones already extended with command line
params.
Change FillDiskCommandAction to try to stop the remote process if the
command failed with an exception.
Signed-off-by: stack <st...@apache.org>
---
.../org/apache/hadoop/hbase/IntegrationTestBase.java | 3 ++-
.../hbase/chaos/actions/FillDiskCommandAction.java | 16 ++++++++++------
.../hadoop/hbase/chaos/factories/MonkeyConstants.java | 4 ++--
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
index 8126d0b..903cf42 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
@@ -92,7 +92,7 @@ public abstract class IntegrationTestBase extends AbstractHBaseTool {
// Add entries for the CM from hbase-site.xml as a convenience.
// Do this prior to loading from the properties file to make sure those in the properties
// file are given precedence to those in hbase-site.xml (backwards compatibility).
- loadMonkeyProperties(monkeyProps, HBaseConfiguration.create());
+ loadMonkeyProperties(monkeyProps, conf);
if (cmd.hasOption(CHAOS_MONKEY_PROPS)) {
String chaosMonkeyPropsFile = cmd.getOptionValue(CHAOS_MONKEY_PROPS);
if (StringUtils.isNotEmpty(chaosMonkeyPropsFile)) {
@@ -183,6 +183,7 @@ public abstract class IntegrationTestBase extends AbstractHBaseTool {
if (fact == null) {
fact = getDefaultMonkeyFactory();
}
+ LOG.info("Using chaos monkey factory: {}", fact.getClass());
monkey = fact.setUtil(util)
.setTableName(getTablename())
.setProperties(monkeyProps)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
index b7af31f..f5ca1d7 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
@@ -58,15 +58,15 @@ public class FillDiskCommandAction extends SudoCommandAction {
String hostname = server.getHostname();
try {
- clusterManager.execSudoWithRetries(hostname, timeout, getFillCommand());
- Thread.sleep(duration);
- } catch (InterruptedException e) {
- LOG.debug("Failed to run the command for the full duration", e);
+ clusterManager.execSudo(hostname, duration, getFillCommand());
+ } catch (IOException ex) {
+ LOG.info("Potential timeout. We try to stop the dd process on target machine");
+ clusterManager.execSudoWithRetries(hostname, timeout, getStopCommand());
+ throw ex;
} finally {
clusterManager.execSudoWithRetries(hostname, timeout, getClearCommand());
+ LOG.info("Finished to execute FillDiskCommandAction");
}
-
- LOG.info("Finished to execute FillDiskCommandAction");
}
private String getFillCommand(){
@@ -80,4 +80,8 @@ public class FillDiskCommandAction extends SudoCommandAction {
private String getClearCommand(){
return String.format("rm -f %s/garbage", path);
}
+
+ private String getStopCommand() {
+ return String.format("killall dd");
+ }
}
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
index f4c34b5..7fbdd1f 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
@@ -55,7 +55,7 @@ public interface MonkeyConstants {
String NETWORK_ISSUE_RATIO = "network.issue.ratio";
String NETWORK_ISSUE_DELAY = "network.issue.delay";
String NETWORK_ISSUE_INTERFACE = "network.issue.interface";
- //should be big enough to create the file
+ //should be higher than the usual timeout because the target machine might respond slowly
String FILL_DISK_COMMAND_TIMEOUT = "fill.disk.command.timeout";
String FILL_DISK_PATH = "fill.disk.path";
String FILL_DISK_FILE_SIZE = "fill.disk.file.size";
@@ -101,7 +101,7 @@ public interface MonkeyConstants {
float DEFAULT_NETWORK_ISSUE_RATIO = 0.1f;
long DEFAULT_NETWORK_ISSUE_DELAY = 100;
String DEFAULT_NETWORK_ISSUE_INTERFACE = "eth0";
- long DEFAULT_FILL_DISK_COMMAND_TIMEOUT = 5 * 60 * 1000 + 30 * 1000;//duration + timeout
+ long DEFAULT_FILL_DISK_COMMAND_TIMEOUT = 60 * 1000;
String DEFAULT_FILL_DISK_PATH = "/tmp";
long DEFAULT_FILL_DISK_FILE_SIZE = 0;
long DEFAULT_FILL_DISK_ISSUE_DURATION = 5 * 60 * 1000;