You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/06 20:26:17 UTC
[2/8] git commit: SLIDER-296 ContainerReleaseSelectors have the job
of choosing containers to release.
SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ce953ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ce953ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ce953ca
Branch: refs/heads/develop
Commit: 5ce953ca301a90e27b1cbe193a482a9cb83c3084
Parents: 32fd3f5
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 16:58:27 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 16:58:27 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/Comparators.java | 58 ++++++
.../providers/AbstractProviderService.java | 18 +-
.../slider/providers/ProviderService.java | 12 +-
.../server/appmaster/SliderAppMaster.java | 4 +-
.../slider/server/appmaster/state/AppState.java | 134 +++++++++----
.../state/ContainerReleaseSelector.java | 38 ++++
.../MostRecentContainerReleaseSelector.java | 52 +++++
.../appmaster/state/SimpleReleaseSelector.java | 34 ++++
.../TestAppStateContainerFailure.groovy | 166 ----------------
.../appstate/TestAppStateDynamicRoles.groovy | 93 ---------
.../TestAppStateRebuildOnAMRestart.groovy | 124 ------------
.../appstate/TestAppStateRolePlacement.groovy | 99 ----------
.../appstate/TestAppStateRoleRelease.groovy | 82 --------
.../TestContainerResourceAllocations.groovy | 108 -----------
.../model/appstate/TestFlexDynamicRoles.groovy | 189 ------------------
.../TestMockAppStateContainerFailure.groovy | 166 ++++++++++++++++
.../TestMockAppStateDynamicRoles.groovy | 94 +++++++++
.../TestMockAppStateFlexDynamicRoles.groovy | 190 +++++++++++++++++++
.../appstate/TestMockAppStateFlexing.groovy | 126 ++++++++++++
.../TestMockAppStateRMOperations.groovy | 176 +++++++++++++++++
.../TestMockAppStateRebuildOnAMRestart.groovy | 124 ++++++++++++
.../TestMockAppStateRolePlacement.groovy | 99 ++++++++++
.../appstate/TestMockAppStateRoleRelease.groovy | 82 ++++++++
.../TestMockContainerResourceAllocations.groovy | 108 +++++++++++
.../model/appstate/TestMockFlexing.groovy | 128 -------------
.../model/appstate/TestMockRMOperations.groovy | 176 -----------------
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 56 +++---
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 3 +-
30 files changed, 1510 insertions(+), 1234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 0000000..0ccca0f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ long result = o1 - o2;
+ // need to comparisons with a diff greater than integer size
+ if (result < 0 ) {
+ return -1;
+ } else if (result >0) {
+ return 1;
+ }
+ return 0;
+ }
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7639625..7483a8d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -32,6 +32,8 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
+import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -318,11 +320,21 @@ public abstract class AbstractProviderService
}
@Override
public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException {
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData) throws IOException {
this.amWebAPI = unsecureWebAPI;
this.registryInstanceData = registryInstanceData;
}
+
+ /**
+ * {@inheritDoc}
+ *
+ *
+ * @return The base implementation returns the most recent containers first.
+ */
+ @Override
+ public ContainerReleaseSelector createContainerReleaseSelector() {
+ return new MostRecentContainerReleaseSelector();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index b9fa34c..4278b09 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -33,6 +33,7 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -172,6 +173,13 @@ public interface ProviderService extends ProviderCore, Service,
*/
void applyInitialRegistryDefinitions(URL unsecureWebAPI,
URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException;
+ ServiceInstanceData registryInstanceData)
+ throws IOException;
+
+ /**
+ * Create the container release selector for this provider...any policy
+ * can be implemented
+ * @return the selector to use for choosing containers.
+ */
+ ContainerReleaseSelector createContainerReleaseSelector();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b3650ca..dbf08e1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -105,6 +105,7 @@ import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
@@ -677,7 +678,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
fs.getFileSystem(),
historyDir,
liveContainers,
- appInformation);
+ appInformation,
+ new SimpleReleaseSelector());
// add the AM to the list of nodes in the cluster
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 08d43db..c20c924 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,7 +18,9 @@
package org.apache.slider.server.appmaster.state;
+import com.beust.jcommander.internal.Lists;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -63,7 +65,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -251,6 +255,8 @@ public class AppState {
private int failureThreshold = 10;
private String logServerURL = "";
+
+ private ContainerReleaseSelector containerReleaseSelector;
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -440,22 +446,27 @@ public class AppState {
* @param historyDir directory containing history files
* @param liveContainers list of live containers supplied on an AM restart
* @param applicationInfo
+ * @param releaseSelector
*/
public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration appmasterConfig, Configuration publishedProviderConf,
+ Configuration appmasterConfig,
+ Configuration publishedProviderConf,
List<ProviderRole> providerRoles,
FileSystem fs,
Path historyDir,
List<Container> liveContainers,
- Map<String, String> applicationInfo) throws
- BadClusterStateException,
- BadConfigException,
- IOException {
+ Map<String, String> applicationInfo,
+ SimpleReleaseSelector releaseSelector)
+ throws BadClusterStateException, BadConfigException, IOException {
+ Preconditions.checkArgument(instanceDefinition != null);
+ Preconditions.checkArgument(releaseSelector != null);
+
this.publishedProviderConf = publishedProviderConf;
- this.applicationInfo = applicationInfo != null ? applicationInfo
- : new HashMap<String, String>();
+ this.applicationInfo = applicationInfo != null ? applicationInfo
+ : new HashMap<String, String>();
clientProperties = new HashMap<>();
+ containerReleaseSelector = releaseSelector;
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
@@ -465,8 +476,8 @@ public class AppState {
String val = publishedProviderConf.get(key);
clientProperties.put(key, val);
}
-
-
+
+
// set the cluster specification (once its dependency the client properties
// is out the way
@@ -479,15 +490,16 @@ public class AppState {
}
ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
+ instanceDefinition.getResourceOperations();
+
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
log.info("Adding new role {}", name);
MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
+ ProviderRole dynamicRole =
+ createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
providerRoles.add(dynamicRole);
}
@@ -498,24 +510,24 @@ public class AppState {
//set the livespan
MapOperations globalInternalOpts =
- instanceDefinition.getInternalOperations().getGlobalOptions();
+ instanceDefinition.getInternalOperations().getGlobalOptions();
startTimeThreshold = globalInternalOpts.getOptionInt(
- OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
- OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
-
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+
failureThreshold = globalInternalOpts.getOptionInt(
- OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
- OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
initClusterStatus();
// add the roles
roleHistory = new RoleHistory(providerRoles);
roleHistory.onStart(fs, historyDir);
-
+
//rebuild any live containers
rebuildModelFromRestart(liveContainers);
-
+
// any am config options to pick up
logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL,
@@ -845,6 +857,27 @@ public class AppState {
return nodes;
}
+
+ /**
+ * enum nodes by role ID, from either the active or live node list
+ * @param roleId role the container must be in
+ * @param active flag to indicate "use active list" rather than the smaller
+ * "live" list
+ * @return a list of nodes, may be empty
+ */
+ public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
+ boolean active) {
+ List<RoleInstance> nodes = new ArrayList<>();
+ Collection<RoleInstance> allRoleInstances;
+ allRoleInstances = active? activeContainers.values() : liveNodes.values();
+ for (RoleInstance node : allRoleInstances) {
+ if (node.roleId == roleId) {
+ nodes.add(node);
+ }
+ }
+ return nodes;
+ }
+
/**
* Build an instance map.
@@ -911,19 +944,19 @@ public class AppState {
throws SliderInternalStateException {
ContainerId id = container.getId();
//look up the container
- RoleInstance info = getActiveContainer(id);
- if (info == null) {
+ RoleInstance instance = getActiveContainer(id);
+ if (instance == null) {
throw new SliderInternalStateException(
- "No active container with ID " + id.toString());
+ "No active container with ID " + id);
}
//verify that it isn't already released
if (containersBeingReleased.containsKey(id)) {
throw new SliderInternalStateException(
"Container %s already queued for release", id);
}
- info.released = true;
- containersBeingReleased.put(id, info.container);
- RoleStatus role = lookupRoleStatus(info.roleId);
+ instance.released = true;
+ containersBeingReleased.put(id, instance.container);
+ RoleStatus role = lookupRoleStatus(instance.roleId);
role.incReleasing();
roleHistory.onContainerReleaseSubmitted(container);
}
@@ -1229,8 +1262,8 @@ public class AppState {
if (shortLived) {
roleStatus.incStartFailed();
}
-
- if (failedContainer!= null) {
+
+ if (failedContainer != null) {
roleHistory.onFailedContainer(failedContainer, shortLived);
}
@@ -1485,18 +1518,43 @@ public class AppState {
// get the nodes to release
int roleId = role.getKey();
- List<NodeInstance> nodesForRelease =
- roleHistory.findNodesForRelease(roleId, excess);
-
- for (NodeInstance node : nodesForRelease) {
- RoleInstance possible = findRoleInstanceOnHost(node, roleId);
- if (possible == null) {
- throw new SliderInternalStateException(
- "Failed to find a container to release on node %s", node.hostname);
+
+ // enum all active nodes that aren't being released
+ List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true);
+
+ // cut all release-in-progress nodes
+ ListIterator<RoleInstance> li = containersToRelease.listIterator();
+ while (li.hasNext()) {
+ RoleInstance next = li.next();
+ if (next.released) {
+ li.remove();
}
- containerReleaseSubmitted(possible.container);
- operations.add(new ContainerReleaseOperation(possible.getId()));
+ }
+
+ // warn if the desired state can't be reaced
+ if (containersToRelease.size() < excess) {
+ log.warn("Not enough nodes to release...short of {} nodes",
+ containersToRelease.size() - excess);
+ }
+
+ // ask the release selector to sort the targets
+ containersToRelease = containerReleaseSelector.sortCandidates(
+ roleId,
+ containersToRelease,
+ excess);
+
+ //crop to the excess
+ List<RoleInstance> finalCandidates = (excess < containersToRelease.size())
+ ? containersToRelease.subList(0, excess)
+ : containersToRelease;
+
+
+ // then build up a release operation, logging each container as released
+ for (RoleInstance possible : finalCandidates) {
+ log.debug("Targeting for release: {}", possible);
+ containerReleaseSubmitted(possible.container);
+ operations.add(new ContainerReleaseOperation(possible.getId()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
new file mode 100644
index 0000000..0cbc134
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.List;
+
+/**
+ * Interface implemented by anything that must choose containers to release
+ *
+ */
+public interface ContainerReleaseSelector {
+
+ /**
+ * Given a list of candidate containers, return a sorted version of the priority
+ * in which they should be released.
+ * @param candidates candidate list ... everything considered suitable
+ * @return
+ */
+ List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect);
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
new file mode 100644
index 0000000..9d936a1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.slider.common.tools.Comparators;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Sort the candidate list by the most recent container first.
+ */
+public class MostRecentContainerReleaseSelector implements ContainerReleaseSelector {
+
+ @Override
+ public List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect) {
+ Collections.sort(candidates, new newerThan());
+ return candidates;
+ }
+
+ private static class newerThan implements Comparator<RoleInstance>, Serializable {
+ private final Comparator<Long> innerComparator =
+ new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
+ public int compare(RoleInstance o1, RoleInstance o2) {
+ return innerComparator.compare(o1.createTime, o2.createTime);
+
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
new file mode 100644
index 0000000..b7f0e05
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.List;
+
+/**
+ * Simplest release selector simply returns the list
+ */
+public class SimpleReleaseSelector implements ContainerReleaseSelector {
+
+ @Override
+ public List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect) {
+ return candidates;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
deleted file mode 100644
index 9c17763..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateContainerFailure extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateContainerFailure"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8000, 4)
- }
-
- @Test
- public void testShortLivedFail() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
- long created = instance.createTime
- long started = instance.startTime
- assert created > 0
- assert started >= created
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- assert appState.isShortLived(instance)
- AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1))
- assert result.roleInstance != null
- assert result.containerFailed
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 1
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
- assert queue.size() == 0
-
- }
-
- @Test
- public void testLongLivedFail() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
- instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
- assert !appState.isShortLived(instance)
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- containerStatus(cid, 1))
- assert result.roleInstance != null
- assert result.containerFailed
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 0
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
- assert queue.size() == 1
-
- }
-
- @Test
- public void testNodeStartFailure() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndSubmitNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
-
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 1
-
-
- RoleHistory history = appState.roleHistory
- history.dump();
- List<NodeInstance> queue = history.cloneAvailableList(0)
- assert queue.size() == 0
-
- NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
- NodeEntry re = ni.get(0)
- assert re.failed == 1
- assert re.startFailed == 1
- }
-
- @Test
- public void testRecurrentStartupFailure() throws Throwable {
-
- role0Status.desired = 1
- try {
- for (int i = 0; i< 100; i++) {
- List<RoleInstance> instances = createAndSubmitNodes()
- assert instances.size() == 1
-
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- log.info("$i instance $instances[0] $cid")
- assert cid
- appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
- AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid))
- assert result.containerFailed
- }
- fail("Cluster did not fail from too many startup failures")
- } catch (TriggerClusterTeardownException teardown) {
- log.info("Exception $teardown.exitCode : $teardown")
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
deleted file mode 100644
index 6d70885..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.RoleInstance
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateDynamicRoles"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Override
- void initApp() {
- super.initApp()
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0,0,0)
-
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "4",
- ]
-
- instance.resourceOperations.components["dynamic"]= opts
-
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null)
- }
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
-
- List<RoleInstance> instances = createAndStartNodes()
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- appState.getRoleHistory().dump();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
deleted file mode 100644
index 8d7ff4f..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRebuildOnAMRestart"
- }
-
- @Test
- public void testRebuild() throws Throwable {
-
- int r0 = 1
- int r1 = 2
- int r2 = 3
- role0Status.desired = r0
- role1Status.desired = r1
- role2Status.desired = r2
- List<RoleInstance> instances = createAndStartNodes()
-
- int clusterSize = r0 + r1 + r2
- assert instances.size() == clusterSize
-
- //clone the list
- List<RoleInstance> cloned = [];
- List<Container> containers = []
- instances.each { RoleInstance elt ->
- cloned.add(elt.clone() as RoleInstance)
- containers.add(elt.container)
- }
- NodeMap nodemap = appState.roleHistory.cloneNodemap()
-
- // now destroy the app state
- appState = new AppState(new MockRecordFactory())
-
- //and rebuild
- appState.buildInstance(
- factory.newInstanceDefinition(r0, r1, r2),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- containers,
- null)
-
- assert appState.getStartedCountainerCount() == clusterSize
-
- appState.getRoleHistory().dump();
-
- //check that the app state direct structures match
- List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0)
- List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1)
- List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2)
-
- assert r0 == r0live.size()
- assert r1 == r1live.size()
- assert r2 == r2live.size()
-
- //now examine the role history
- NodeMap newNodemap = appState.roleHistory.cloneNodemap()
-
- for (NodeInstance nodeInstance : newNodemap.values()) {
- String hostname = nodeInstance.hostname
- NodeInstance orig = nodemap[hostname]
- assertNotNull("Null entry in original nodemap for " + hostname, orig)
-
- for (int i = 0; i < ROLE_COUNT; i++) {
-
- assert (nodeInstance.getActiveRoleInstances(i) ==
- orig.getActiveRoleInstances(i))
- NodeEntry origRE = orig.getOrCreate(i)
- NodeEntry newRE = nodeInstance.getOrCreate(i)
- assert origRE.live == newRE.live
- assert newRE.starting == 0
- }
- }
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
-
- def status = appState.getClusterStatus()
- // verify the AM restart container count was set
- String restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
- assert restarted != null;
- //and that the count == 1 master + the region servers
- assert Integer.parseInt(restarted) == containers.size()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
deleted file mode 100644
index fba1ea0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
-/**
- * Test that the app state lets you ask for nodes, get a specific host,
- * release it and then get that one back again.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRolePlacement extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRolePlacement"
- }
-
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request = operation.request
- Container allocated = engine.allocateContainer(request)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated([(Container)allocated], assignments, operations)
- assert operations.size() == 0
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container container = assigned.container
- assert container.id == allocated.id
- int roleId = assigned.role.priority
- assert roleId == extractRole(request.priority)
- assert assigned.role.name == ROLE0
- String containerHostname = RoleHistoryUtils.hostnameOf(container);
- RoleInstance ri = roleInstance(assigned)
- //tell the app it arrived
- appState.containerStartSubmitted(container, ri);
- assert appState.onNodeManagerContainerStarted(container.id)
- assert role0Status.started == 1
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
-
- //now it is surplus
- role0Status.desired = 0
- ops = appState.reviewRequestAndReleaseNodes()
- ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
-
- assert release.containerId == container.id
- engine.execute(ops)
- assert appState.onCompletedNode(containerStatus(container)).roleInstance
-
- //view the world
- appState.getRoleHistory().dump();
-
- //now ask for a new one
- role0Status.desired = 1
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request2 = operation.request
- assert request2 != null
- assert request2.nodes[0] == containerHostname
- engine.execute(ops)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
deleted file mode 100644
index f087a30..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.RoleInstance
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRoleRelease extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRolePlacement"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
- /**
- * Allocate to all nodes
- */
- role0Status.desired = 6
- role1Status.desired = 5
- role2Status.desired = 4
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 15
-
- //now it is surplus
- role0Status.desired = 0
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
-
- List<ContainerId> released = []
- engine.execute(ops, released)
- List<ContainerId> ids = extractContainerIds(instances, 0)
- released.each { ContainerId cid ->
- assert appState.onCompletedNode(containerStatus(cid)).roleInstance
- assert ids.contains(cid)
- }
-
- //view the world
- appState.getRoleHistory().dump();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
deleted file mode 100644
index a0b1100..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.core.conf.ConfTree
-import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
-import org.junit.Test
-
-/**
- * Test the container resource allocation logic
- */
-@CompileStatic
-@Slf4j
-class TestContainerResourceAllocations extends BaseMockAppStateTest {
-
- @Override
- String getTestName() {
- "TestContainerResourceAllocations"
- }
-
- @Test
- public void testNormalAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
-
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == 512
- assert requirements.virtualCores == 2
- }
-
- @Test
- public void testMaxMemAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
-
- cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
- ResourceKeys.YARN_RESOURCE_MAX)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == RM_MAX_RAM
- assert requirements.virtualCores == 2
- }
-
- @Test
- public void testMaxCoreAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
- 512)
- cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES,
- ResourceKeys.YARN_RESOURCE_MAX)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == 512
- assert requirements.virtualCores == RM_MAX_CORES
- }
-
- @Test
- public void testMaxDefaultAllocations() throws Throwable {
-
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == ResourceKeys.DEF_YARN_MEMORY
- assert requirements.virtualCores == ResourceKeys.DEF_YARN_CORES
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
deleted file mode 100644
index 911880a..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.core.exceptions.BadConfigException
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestFlexDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateDynamicRoles"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Override
- void initApp() {
- super.initApp()
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0, 0, 0)
-
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
- ]
-
- instance.resourceOperations.components["dynamic"] = opts
-
-
- appState.buildInstance(instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null)
- }
-
-
- private ConfTreeOperations init() {
- createAndStartNodes();
- def resources = appState.instanceDefinition.resources;
- return new ConfTreeOperations(resources)
- }
-
- @Test
- public void testDynamicFlexAddRole() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- appState.lookupRoleStatus(7)
- appState.lookupRoleStatus(6)
- //gaps are still there
- try {
- assert null == appState.lookupRoleStatus(5)
- } catch (RuntimeException expected) {
- }
- }
-
- @Test
- public void testDynamicFlexAddRoleConflictingPriority() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
- ]
-
- cd.components["role4"] = opts
- try {
- appState.updateResourceDefinitions(cd.confTree);
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- fail("Expected an exception")
- } catch (BadConfigException expected) {
- }
- }
-
- @Test
- public void testDynamicFlexDropRole() throws Throwable {
- def cd = init()
- cd.components.remove("dynamic");
- appState.updateResourceDefinitions(cd.confTree);
-
- def getCD = appState.getClusterStatus()
- dumpClusterDescription("updated CD", getCD)
- //status is retained for future
- appState.lookupRoleStatus(6)
- }
-
-
- @Test
- public void testHistorySaveFlexLoad() throws Throwable {
- def cd = init()
- def roleHistory = appState.roleHistory
- Path history = roleHistory.saveHistory(0x0001)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- historyWriter.read(fs, history, appState.roleHistory)
- }
-
- @Test
- public void testHistoryFlexSaveLoad() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- def roleHistory = appState.roleHistory
- Path history = roleHistory.saveHistory(0x0002)
- //now reset the app state
- def historyWorkDir2 = new File("target/history" + testName + "-0002")
- def historyPath2 = new Path(historyWorkDir2.toURI())
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath2,
- null, null)
- historyWriter.read(fs, history, appState.roleHistory)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
new file mode 100644
index 0000000..3a287bd
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateContainerFailure extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateContainerFailure"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8000, 4)
+ }
+
+ @Test
+ public void testShortLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ long created = instance.createTime
+ long started = instance.startTime
+ assert created > 0
+ assert started >= created
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ assert appState.isShortLived(instance)
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ }
+
+ @Test
+ public void testLongLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
+ assert !appState.isShortLived(instance)
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ AppState.NodeCompletionResult result = appState.onCompletedNode(
+ containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 0
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 1
+
+ }
+
+ @Test
+ public void testNodeStartFailure() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+
+ RoleHistory history = appState.roleHistory
+ history.dump();
+ List<NodeInstance> queue = history.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
+ NodeEntry re = ni.get(0)
+ assert re.failed == 1
+ assert re.startFailed == 1
+ }
+
+ @Test
+ public void testRecurrentStartupFailure() throws Throwable {
+
+ role0Status.desired = 1
+ try {
+ for (int i = 0; i< 100; i++) {
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ log.info("$i instance $instances[0] $cid")
+ assert cid
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid))
+ assert result.containerFailed
+ }
+ fail("Cluster did not fail from too many startup failures")
+ } catch (TriggerClusterTeardownException teardown) {
+ log.info("Exception $teardown.exitCode : $teardown")
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
new file mode 100644
index 0000000..a204bea
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0,0,0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "4",
+ ]
+
+ instance.resourceOperations.components["dynamic"]= opts
+
+
+ appState.buildInstance(
+ instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null,
+ null, new SimpleReleaseSelector())
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+
+ List<RoleInstance> instances = createAndStartNodes()
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ appState.getRoleHistory().dump();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
new file mode 100644
index 0000000..aa330f8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0, 0, 0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ instance.resourceOperations.components["dynamic"] = opts
+
+
+ appState.buildInstance(instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null, new SimpleReleaseSelector())
+ }
+
+
+ private ConfTreeOperations init() {
+ createAndStartNodes();
+ def resources = appState.instanceDefinition.resources;
+ return new ConfTreeOperations(resources)
+ }
+
+ @Test
+ public void testDynamicFlexAddRole() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ appState.lookupRoleStatus(7)
+ appState.lookupRoleStatus(6)
+ //gaps are still there
+ try {
+ assert null == appState.lookupRoleStatus(5)
+ } catch (RuntimeException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexAddRoleConflictingPriority() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ cd.components["role4"] = opts
+ try {
+ appState.updateResourceDefinitions(cd.confTree);
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ fail("Expected an exception")
+ } catch (BadConfigException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexDropRole() throws Throwable {
+ def cd = init()
+ cd.components.remove("dynamic");
+ appState.updateResourceDefinitions(cd.confTree);
+
+ def getCD = appState.getClusterStatus()
+ dumpClusterDescription("updated CD", getCD)
+ //status is retained for future
+ appState.lookupRoleStatus(6)
+ }
+
+
+ @Test
+ public void testHistorySaveFlexLoad() throws Throwable {
+ def cd = init()
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0001)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+ @Test
+ public void testHistoryFlexSaveLoad() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0002)
+ //now reset the app state
+ def historyWorkDir2 = new File("target/history" + testName + "-0002")
+ def historyPath2 = new Path(historyWorkDir2.toURI())
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath2,
+ null, null, new SimpleReleaseSelector())
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
new file mode 100644
index 0000000..6be158e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.RMOperationHandler
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockFlexing"
+ }
+
+ @Test
+ public void testFlexDuringLaunchPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ //now this is the start point.
+ appState.containerStartSubmitted(target, ri);
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+
+ @Test
+ public void testFlexBeforeAllocationPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert !ops.empty
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ }
+
+
+ @Test
+ public void testFlexDownTwice() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ r0 = 4
+ role0Status.desired = r0
+ List<AppState.NodeCompletionResult> completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 2
+
+ // no-op review
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 0
+
+
+ // now shrink again
+ role0Status.desired = r0 = 1
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 3
+
+ }
+
+
+}