You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/21 16:42:42 UTC
[06/50] [abbrv] git commit: SLIDER-202 monkey has mock tests and is
set to run in AM if enabled
SLIDER-202 monkey has mock tests and is set to run in AM if enabled
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52bd11b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52bd11b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52bd11b5
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 52bd11b5d4d3a17f231e5343c0bfa6c39492fbb3
Parents: 8ed52a9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:06:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:06:11 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 26 +--
.../apache/slider/common/tools/SliderUtils.java | 6 +-
.../apache/slider/core/conf/MapOperations.java | 54 ++++++-
.../providers/agent/AgentClientProvider.java | 8 +-
.../providers/agent/AgentProviderService.java | 7 +-
.../server/appmaster/SliderAppMaster.java | 88 ++++++----
.../appmaster/actions/ActionKillContainer.java | 24 ++-
.../server/appmaster/actions/QueueService.java | 8 +-
.../server/appmaster/monkey/ChaosEntry.java | 12 +-
.../server/appmaster/monkey/ChaosKillAM.java | 12 +-
.../appmaster/monkey/ChaosKillContainer.java | 25 +--
.../appmaster/monkey/ChaosMonkeyService.java | 23 ++-
.../model/mock/MockRMOperationHandler.groovy | 7 +-
.../model/monkey/TestMockMonkey.groovy | 160 +++++++++++++++++++
.../apache/slider/test/SliderTestBase.groovy | 7 +-
15 files changed, 360 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index c2b78cc..3d54140 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -112,34 +112,14 @@ public interface ResourceKeys {
"yarn.container.failure.threshold";
/**
- * Minutes range of the container failure reset window.
- * This is combined with the hour and day values to
- * produce the full window.
+ * prefix for the time of the container failure reset window.
* {@value}
*/
- String CONTAINER_FAILURE_WINDOW_MINUTES =
- "yarn.container.failure.window.minutes";
-
-
- /**
- * Hours range of the container failure reset window
- * This is combined with the minute and day values to
- * produce the full window.
- * {@value}
- */
- String CONTAINER_FAILURE_WINDOW_HOURS =
- "yarn.container.failure.window.hours";
+ String CONTAINER_FAILURE_WINDOW =
+ "yarn.container.failure.window";
- /**
- * Hours range of the container failure reset window
- * This is combined with the hour and minute values to
- * produce the full window.
- * {@value}
- */
- String CONTAINER_FAILURE_WINDOW_DAYS =
- "yarn.container.failure.window.days";
int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index edbf2b2..d7f159c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
@@ -1303,8 +1303,8 @@ public final class SliderUtils {
SliderException, IOException {
Path imagePath;
String imagePathOption =
- internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+ internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index bb57b94..98a51bc 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.conf;
+import com.google.common.base.Preconditions;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.slf4j.Logger;
@@ -61,12 +62,11 @@ public class MapOperations implements Map<String, String> {
this.name = name;
}
-
/**
- * Get a cluster option or value
+ * Get an option value
*
- * @param key
- * @param defVal
+ * @param key key
+ * @param defVal default value
* @return option in map or the default
*/
public String getOption(String key, String defVal) {
@@ -74,14 +74,27 @@ public class MapOperations implements Map<String, String> {
return val != null ? val : defVal;
}
+ /**
+ * Get a boolean option
+ *
+ * @param key option key
+ * @param defVal default value
+ * @return option true if the option equals "true", or the default value
+ * if the option was not defined at all.
+ */
+ public Boolean getOptionBool(String key, boolean defVal) {
+ String val = getOption(key, Boolean.toString(defVal));
+ return Boolean.valueOf(val);
+ }
/**
* Get a cluster option or value
*
- * @param key
+ * @param key option key
* @return the value
* @throws BadConfigException if the option is missing
*/
+
public String getMandatoryOption(String key) throws BadConfigException {
String val = options.get(key);
if (val == null) {
@@ -247,4 +260,35 @@ public class MapOperations implements Map<String, String> {
}
return builder.toString();
}
+
+ /**
+ * Get the time range of a set of keys
+ * @param basekey
+ * @param defDays
+ * @param defHours
+ * @param defMins
+ * @param defSecs
+ * @return
+ */
+ public long getTimeRange(String basekey,
+ int defDays,
+ int defHours,
+ int defMins,
+ int defSecs) {
+ Preconditions.checkArgument(basekey != null);
+ int days = getOptionInt(basekey + ".days", defDays);
+ int hours = getOptionInt(basekey + ".hours", defHours);
+
+ int minutes = getOptionInt(basekey + ".minutes", defMins);
+ int seconds = getOptionInt(basekey + ".seconds", defSecs);
+ // range check
+ Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
+ && seconds >= 0,
+ "Time range for %s has negative time component %s:%s:%s:%s",
+ basekey, days, hours, minutes, seconds);
+
+ // calculate total time, schedule the reset if expected
+ long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
+ return totalMinutes * 60 + seconds;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 2ce16c7..72b0a42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -21,7 +21,7 @@ package org.apache.slider.providers.agent;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -109,7 +109,7 @@ public class AgentClientProvider extends AbstractClientProvider
getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
if (SliderUtils.isUnset(appHome)) {
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
sliderFileSystem.verifyFileExists(new Path(agentImage));
}
}
@@ -174,12 +174,12 @@ public class AgentClientProvider extends AbstractClientProvider
String appHome = instanceDefinition.getAppConfOperations().
getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (SliderUtils.isUnset(appHome) && SliderUtils.isUnset(agentImage)) {
throw new BadConfigException("Either agent package path " +
AgentKeys.PACKAGE_PATH + " or image root " +
- OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH
+ " must be provided.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 59a6e29..0c57b79 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.SliderKeys;
@@ -62,8 +63,6 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -265,7 +264,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("PYTHONPATH set to {}", pythonPath);
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (agentImage != null) {
LocalResource agentImageRes = fileSystem.createAmResource(new Path(agentImage), LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
@@ -1169,7 +1168,7 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
- .getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+ .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
return tokens;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6c7aba..1642cc5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -1195,31 +1196,19 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ResetFailureWindow reset = new ResetFailureWindow();
ConfTreeOperations ops = new ConfTreeOperations(resources);
MapOperations globals = ops.getGlobalOptions();
- int days = globals.getOptionInt(ResourceKeys.CONTAINER_FAILURE_WINDOW_DAYS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS);
- int hours = globals.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_WINDOW_HOURS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS);
- int minutes = globals.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_WINDOW_MINUTES,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES);
- // range check
- if (days < 0 || hours < 0 || minutes < 0) {
- throw new BadConfigException("Failure window contains negative values:"
- + "days=%d hours=%d, minutes=%d",
- days, hours, minutes);
- }
- // calculate total time, schedule the reset if expected
- int totalMinutes = days * 24 * 60 + hours * 24 + minutes;
- if (totalMinutes > 0) {
+ long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0);
+ if (seconds > 0) {
log.info(
- "Scheduling the failure window reset interval to every {} minutes",
- totalMinutes);
+ "Scheduling the failure window reset interval to every {} seconds",
+ seconds);
RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
- reset, totalMinutes, totalMinutes, TimeUnit.MINUTES, 0);
+ reset, seconds, seconds, TimeUnit.SECONDS, 0);
actionQueues.renewing("failures", renew);
} else {
- log.warn("Failure window reset interval is not set");
+ log.info("Failure window reset interval is not set");
}
}
@@ -1458,7 +1447,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//throws NoSuchNodeException if it is missing
RoleInstance instance =
appState.getLiveInstanceByContainerID(containerID);
- queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS));
+ queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS,
+ rmOperationHandler));
Messages.KillContainerResponseProto.Builder builder =
Messages.KillContainerResponseProto.newBuilder();
builder.setSuccess(true);
@@ -1469,6 +1459,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
rmOperationHandler.execute(operations);
}
+
+ /**
+ * Get the RM operations handler for direct scheduling of work.
+ */
+ @VisibleForTesting
+ public RMOperationHandler getRmOperationHandler() {
+ return rmOperationHandler;
+ }
+
@Override
public Messages.AMSuicideResponseProto amSuicide(
Messages.AMSuicideRequestProto request)
@@ -1742,18 +1741,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
public boolean maybeStartMonkey() {
+ MapOperations internals = getGlobalInternalOptions();
+
+ Boolean enabled =
+ internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+ if (!enabled) {
+ log.info("Chaos monkey disabled");
+ }
+
+ long monkeyInterval = internals.getTimeRange(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+ 0);
+ log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
+ monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
- int amKillProbability = 100;
- int containerKillProbability = 200;
- monkey.addTarget("AM killer",
- new ChaosKillAM(this, actionQueues, -1),
- amKillProbability);
- monkey.addTarget("Container killer",
- new ChaosKillContainer(this, actionQueues),
- amKillProbability);
+ int amKillProbability = internals.getOptionInt(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
+ if (amKillProbability > 0) {
+ log.info("Adding AM killer with probability %f", amKillProbability/100.0);
+ monkey.addTarget("AM killer",
+ new ChaosKillAM(actionQueues, -1), amKillProbability
+ );
+ }
+ int containerKillProbability = internals.getOptionInt(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
+ if (containerKillProbability > 0) {
+ monkey.addTarget("Container killer",
+ new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
+ amKillProbability
+ );
+ }
initAndAddService(monkey);
// and schedule it
- schedule(monkey.getChaosAction(60, TimeUnit.SECONDS));
+ schedule(monkey.getChaosAction(monkeyInterval, TimeUnit.SECONDS));
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 7668bd5..95edca4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -18,10 +18,12 @@
package org.apache.slider.server.appmaster.actions;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.AppState;
import java.util.LinkedList;
@@ -31,13 +33,27 @@ import java.util.concurrent.TimeUnit;
public class ActionKillContainer extends AsyncAction {
private final ContainerId containerId;
-
- public ActionKillContainer(ContainerId containerId, long delay,
- TimeUnit timeUnit) {
+ private final RMOperationHandler operationHandler;
+ public ActionKillContainer(
+ ContainerId containerId,
+ long delay,
+ TimeUnit timeUnit,
+ RMOperationHandler operationHandler) {
super("kill container", delay, timeUnit);
+ this.operationHandler = operationHandler;
+ Preconditions.checkArgument(containerId != null);
+
this.containerId = containerId;
}
+ /**
+ * Get the container ID to kill
+ * @return
+ */
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
@@ -46,6 +62,6 @@ public class ActionKillContainer extends AsyncAction {
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
- appMaster.executeRMOperations(opsList);
+ operationHandler.execute(opsList);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 1154a44..ac59096 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -55,18 +55,14 @@ implements Runnable, QueueAccess {
/**
* Immediate actions.
- * Package scoped for testing.
*/
- @VisibleForTesting
- final BlockingDeque<AsyncAction> actionQueue =
+ public final BlockingDeque<AsyncAction> actionQueue =
new LinkedBlockingDeque<>();
/**
* Actions to be scheduled in the future
- * Package scoped for testing.
*/
- @VisibleForTesting
- final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
/**
* Map of renewing actions by name ... this is to allow them to
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 13bda57..5905d2f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -41,7 +41,7 @@ public class ChaosEntry {
/**
- * Constructor -includes validaton of all arguments
+ * Constructor -includes validation of all arguments
* @param name
* @param target
* @param probability
@@ -51,7 +51,8 @@ public class ChaosEntry {
Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability < 10000, "probability over 100%");
+ Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
+ "probability over 100%");
this.name = name;
this.target = target;
this.probability = probability;
@@ -71,11 +72,12 @@ public class ChaosEntry {
/**
* Invoke Chaos if the trigger value is in range of the probability
- * @param trigger trigger value, 0-10K
+ * @param value trigger value, 0-10K
* @return true if the chaos method was invoked
*/
- public boolean maybeInvokeChaos(long trigger) {
- if (probability < 0) {
+ public boolean maybeInvokeChaos(long value) {
+ log.debug("Probability {} trigger={}", probability, value);
+ if (value < probability) {
invokeChaos();
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
index c464ac3..3c1a914 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
@@ -18,21 +18,21 @@
package org.apache.slider.server.appmaster.monkey;
-import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import java.util.concurrent.TimeUnit;
+/**
+ * Kill the AM
+ */
public class ChaosKillAM implements ChaosTarget {
- private final SliderAppMaster appMaster;
+ public static final int DELAY = 1000;
private final QueueAccess queues;
private final int exitCode;
- public ChaosKillAM(SliderAppMaster appMaster,
- QueueAccess queues, int exitCode) {
- this.appMaster = appMaster;
+ public ChaosKillAM(QueueAccess queues, int exitCode) {
this.queues = queues;
this.exitCode = exitCode;
}
@@ -42,7 +42,7 @@ public class ChaosKillAM implements ChaosTarget {
*/
@Override
public void chaosAction() {
- queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", 1000,
+ queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", DELAY,
TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index 0ffcc8a..daf2590 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -18,9 +18,11 @@
package org.apache.slider.server.appmaster.monkey;
-import org.apache.slider.server.appmaster.SliderAppMaster;
+import com.google.common.base.Preconditions;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
+import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,24 +37,29 @@ import java.util.concurrent.TimeUnit;
public class ChaosKillContainer implements ChaosTarget {
protected static final Logger log =
LoggerFactory.getLogger(ChaosKillContainer.class);
- private final SliderAppMaster appMaster;
+ public static final int DELAY = 100;
+ private final AppState appState;
private final QueueAccess queues;
private final Random random = new Random();
+ private final RMOperationHandler operationHandler;
- public ChaosKillContainer(SliderAppMaster appMaster,
- QueueAccess queues) {
- this.appMaster = appMaster;
+ public ChaosKillContainer(AppState appState,
+ QueueAccess queues,
+ RMOperationHandler operationHandler) {
+ Preconditions.checkNotNull(appState);
+ Preconditions.checkNotNull(queues);
+ this.appState = appState;
this.queues = queues;
+ this.operationHandler = operationHandler;
}
/**
- * Trigger a container kill halt
+ * Trigger a container kill
*/
-
@Override
public void chaosAction() {
List<RoleInstance> liveContainers =
- appMaster.getAppState().cloneLiveContainerInfoList();
+ appState.cloneLiveContainerInfoList();
int size = liveContainers.size();
if (size == 0) {
log.info("No containers to kill");
@@ -63,6 +70,6 @@ public class ChaosKillContainer implements ChaosTarget {
log.info("Killing {}", roleInstance);
queues.schedule(new ActionKillContainer(roleInstance.getId(),
- 100, TimeUnit.MILLISECONDS));
+ DELAY, TimeUnit.MILLISECONDS, operationHandler));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index c525d02..cd4b46b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.monkey;
import com.codahale.metrics.MetricRegistry;
import org.apache.hadoop.service.AbstractService;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.slf4j.Logger;
@@ -28,6 +27,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
@@ -36,8 +36,16 @@ import java.util.concurrent.TimeUnit;
public class ChaosMonkeyService extends AbstractService {
protected static final Logger log =
LoggerFactory.getLogger(ChaosMonkeyService.class);
+ public static final int PERCENT_1 = 100;
+ public static final double PERCENT_1D = 100.0;
+
+ /**
+ * the percentage value as multiplied up
+ */
+ public static final int PERCENT_100 = 100 * PERCENT_1;
private final MetricRegistry metrics;
private final QueueAccess queues;
+ private final Random random = new Random();
private static final List<ChaosEntry> chaosEntries =
new ArrayList<ChaosEntry>();
@@ -50,14 +58,19 @@ public class ChaosMonkeyService extends AbstractService {
public synchronized void addTarget(String name,
- ChaosTarget target,
- long probability) {
-
+ ChaosTarget target, long probability) {
+ log.info("Adding {} with probability {}", name, probability / PERCENT_1);
chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
}
+ /**
+ * Iterate through all the entries and invoke chaos on those wanted
+ */
public void play() {
-
+ for (ChaosEntry chaosEntry : chaosEntries) {
+ long p = random.nextInt(PERCENT_100);
+ chaosEntry.maybeInvokeChaos(p);
+ }
}
public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 8ccd463..0fdba6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -29,17 +29,20 @@ import org.apache.slider.server.appmaster.operations.RMOperationHandler
@Slf4j
class MockRMOperationHandler extends RMOperationHandler {
public List<AbstractRMOperation> operations = [];
-
+ int requests, releases;
+
@Override
public void releaseAssignedContainer(ContainerId containerId) {
operations.add(new ContainerReleaseOperation(containerId))
log.info("Releasing container ID " + containerId.getId())
+ releases++;
}
@Override
public void addContainerRequest(AMRMClient.ContainerRequest req) {
operations.add(new ContainerRequestOperation(req))
log.info("Requesting container role #" + req.priority);
+ requests++;
}
/**
@@ -47,5 +50,7 @@ class MockRMOperationHandler extends RMOperationHandler {
*/
public void clear() {
operations.clear()
+ releases = 0;
+ requests = 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
new file mode 100644
index 0000000..c789011
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.monkey
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.server.appmaster.actions.ActionHalt
+import org.apache.slider.server.appmaster.actions.ActionKillContainer
+import org.apache.slider.server.appmaster.actions.QueueService
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.monkey.ChaosKillAM
+import org.apache.slider.server.appmaster.monkey.ChaosKillContainer
+import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService
+import org.apache.slider.server.appmaster.monkey.ChaosTarget
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestMockMonkey extends BaseMockAppStateTest {
+
+ /**
+ * This queue service is NOT started; tests need to poll the queue
+ * rather than expect them to execute
+ */
+ QueueService queues = new QueueService();
+ ChaosMonkeyService monkey = new ChaosMonkeyService(metricRegistry,
+ queues)
+
+ @Before
+ public void init() {
+ def configuration = new YarnConfiguration()
+ queues.init(configuration)
+ monkey.init(configuration)
+ }
+
+ @Test
+ public void testMonkeyStart() throws Throwable {
+ monkey.start()
+ monkey.stop()
+ }
+
+
+ @Test
+ public void testMonkeyPlay() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
+
+ monkey.play()
+ assert counter.count == 1
+ }
+
+ @Test
+ public void testMonkeyPlaySometimes() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ ChaosCounter counter2 = new ChaosCounter()
+ monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50)
+ monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25)
+
+ for (int i = 0; i < 100; i++) {
+ monkey.play()
+ }
+ log.info("Counter1 = ${counter.count} counter2 = ${counter2.count}")
+ /*
+ * Relying on probability here to give approximate answers
+ */
+ assert counter.count > 25
+ assert counter.count < 75
+ assert counter2.count < counter.count
+ }
+
+ @Test
+ public void testAMKiller() throws Throwable {
+
+ def chaos = new ChaosKillAM(queues, -1)
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 1
+ def action = queues.scheduledActions.take()
+ assert action instanceof ActionHalt
+ }
+
+
+ @Test
+ public void testContainerKillerEmptyApp() throws Throwable {
+
+
+ def chaos = new ChaosKillContainer(appState,
+ queues,
+ new MockRMOperationHandler())
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 0
+ }
+
+
+
+ @Test
+ public void testContainerKiller() throws Throwable {
+ MockRMOperationHandler ops = new MockRMOperationHandler();
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+ def instance = instances[0]
+
+ def chaos = new ChaosKillContainer(appState, queues, ops)
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 1
+ def action = queues.scheduledActions.take()
+ ActionKillContainer killer = (ActionKillContainer) action
+ assert killer.containerId == instance.containerId;
+ killer.execute(null, queues, appState)
+ assert ops.releases == 1;
+
+ ContainerReleaseOperation operation = (ContainerReleaseOperation) ops.operations[0]
+ assert operation.containerId == instance.containerId
+ }
+
+
+
+ /**
+ * Chaos target that just implement a counter
+ */
+ private static class ChaosCounter implements ChaosTarget {
+ int count;
+
+ @Override
+ void chaosAction() {
+ count++;
+ }
+
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ChaosCounter{");
+ sb.append("count=").append(count);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index d7b328f..28b484f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.test
+import com.codahale.metrics.MetricRegistry
import groovy.transform.CompileStatic
import org.apache.hadoop.fs.FileUtil
import org.apache.slider.common.SliderXMLConfKeysForTesting
@@ -38,7 +39,11 @@ import java.nio.file.Files
@CompileStatic
public abstract class SliderTestBase extends SliderTestUtils {
-
+ /**
+ * Singleton metric registry
+ */
+ public static final MetricRegistry metricRegistry = new MetricRegistry()
+
@Rule
public TestName methodName = new TestName();