You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/19 14:22:22 UTC
[08/50] [abbrv] git commit: SLIDER-202: Chaos Monkey adds a (slow)
minicluster test.
SLIDER-202: Chaos Monkey adds a (slow) minicluster test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/53028eaa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/53028eaa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/53028eaa
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 53028eaae347899cef9e695e9ee8b41594c6377b
Parents: 52bd11b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 17:30:49 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 17:30:49 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 49 +++++++---
.../apache/slider/common/tools/SliderUtils.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 39 ++++----
.../TestStandaloneAMMonkeyRestart.groovy | 94 ++++++++++++++++++++
.../TestRegionServerFailureThreshold.groovy | 5 +-
5 files changed, 154 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index 4045c0e..ad384e2 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -84,31 +84,54 @@ public interface InternalKeys {
* Flag to indicate whether or not the chaos monkey is enabled:
* {@value}
*/
- String INTERNAL_CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
- boolean DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED = false;
+ String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
+ boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
/**
* Rate
*/
- String INTERNAL_CHAOS_MONKEY_RATE = "internal.chaos.monkey.rate";
-
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS = 0;
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS = 1;
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES = 0;
+ String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
+ String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
+ String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
+ String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
+ String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
- String INTERNAL_CHAOS_MONKEY_PROBABILITY =
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+
+ /**
+ * Prefix for all chaos monkey probabilities
+ */
+ String CHAOS_MONKEY_PROBABILITY =
"internal.chaos.monkey.probability";
/**
* Probabilies are out of 10000 ; 100==1%
*/
- String INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = INTERNAL_CHAOS_MONKEY_PROBABILITY +".amfailure";
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 10;
- String INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
- INTERNAL_CHAOS_MONKEY_PROBABILITY + ".containerfailure";
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 100;
+ /**
+ * Probability of a monkey check killing the AM: {@value}
+ */
+ String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure";
+
+ /**
+ * Default probability of a monkey check killing the AM: {@value}
+ */
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
+
+ /**
+ * Probability of a monkey check killing a container: {@value}
+ */
+
+ String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".containerfailure";
+
+ /**
+ * Default probability of a monkey check killing the a container: {@value}
+ */
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index d7f159c..7a80f24 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1476,7 +1476,7 @@ public final class SliderUtils {
}
is = new ByteArrayInputStream(content);
} else {
- log.info("Size unknown. Reading {}", zipEntry.getName());
+ log.debug("Size unknown. Reading {}", zipEntry.getName());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
while (true) {
int byteRead = zis.read();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1642cc5..7825e24 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -779,17 +779,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
- sliderAMProvider.start();
+ // chaos monkey
+ maybeStartMonkey();
+ // Start the Slider AM provider
+ sliderAMProvider.start();
-
- // launch the provider; this is expected to trigger a callback that
+ // launch the real provider; this is expected to trigger a callback that
// starts the node review process
launchProviderService(instanceDefinition, confDir);
- // chaos monkey
- maybeStartMonkey();
-
try {
//now block waiting to be told to exit the process
waitForAMCompletionSignal();
@@ -1459,7 +1458,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
rmOperationHandler.execute(operations);
}
-
/**
* Get the RM operations handler for direct scheduling of work.
*/
@@ -1739,42 +1737,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
signalAMComplete(exitCode, exception.toString());
}
-
- public boolean maybeStartMonkey() {
+
+ /**
+ * Start the chaos monkey
+ * @return true if it started
+ */
+ private boolean maybeStartMonkey() {
MapOperations internals = getGlobalInternalOptions();
Boolean enabled =
- internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+ internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
if (!enabled) {
log.info("Chaos monkey disabled");
}
long monkeyInterval = internals.getTimeRange(
- InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+ InternalKeys.CHAOS_MONKEY_INTERVAL,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
0);
log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
int amKillProbability = internals.getOptionInt(
- InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
if (amKillProbability > 0) {
- log.info("Adding AM killer with probability %f", amKillProbability/100.0);
monkey.addTarget("AM killer",
new ChaosKillAM(actionQueues, -1), amKillProbability
);
}
int containerKillProbability = internals.getOptionInt(
- InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
if (containerKillProbability > 0) {
monkey.addTarget("Container killer",
new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
- amKillProbability
+ containerKillProbability
);
}
initAndAddService(monkey);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
new file mode 100644
index 0000000..162bab0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.SleepJob
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testStandaloneAMMonkeyRestart() throws Throwable {
+ describe "Run a Standalone AM with the Chaos monkey set to kill it"
+ // patch the configuration for AM restart
+ int threshold = 2;
+ YarnConfiguration conf = getRestartableConfiguration(threshold)
+
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report
+ report = waitForClusterLive(sliderClient, 30000)
+ describe "Waiting for the cluster to fail"
+ sleep(40000)
+ // end of process
+ report = sliderClient.applicationReport
+ log.info(report.diagnostics)
+ assert report.currentApplicationAttemptId.attemptId == threshold
+ assert YarnApplicationState.FAILED == report.yarnApplicationState
+ assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+ def conf = new YarnConfiguration(configuration)
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+ conf
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
index 1322fd3..b3aaa48 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
@@ -149,8 +149,9 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
}
}
} catch (BadClusterStateException e) {
- assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
- assert e.exitCode == SliderExitCodes.EXIT_BAD_STATE
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_BAD_STATE,
+ ErrorStrings.E_APPLICATION_NOT_RUNNING)
}
ApplicationReport report = client.applicationReport
log.info(report.diagnostics)