You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/23 18:20:17 UTC
[05/50] incubator-slider git commit: SLIDER-966 RoleHistory now
creates the resource requests, issued via the factory. To be honest, I'm not
100% sure I like the current design; I may need to do another iteration
SLIDER-966 RoleHistory now creates the resource requests, issued via the factory. To be honest, I'm not 100% sure I like the current design; I may need to do another iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/aa46b473
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/aa46b473
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/aa46b473
Branch: refs/heads/develop
Commit: aa46b473e11e9ec154ee12cf5d5113d7b974f99c
Parents: ac98d82
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 6 21:19:51 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 6 21:19:51 2015 +0000
----------------------------------------------------------------------
.../appmaster/ProtobufClusterServices.java | 6 +
.../state/AbstractClusterServices.java | 6 +
.../slider/server/appmaster/state/AppState.java | 126 +++++++++++--------
.../server/appmaster/state/RoleHistory.java | 23 ++--
.../appstate/TestMockAppStateAAPlacement.groovy | 29 +++--
.../TestRoleHistoryContainerEvents.groovy | 62 ++++-----
.../TestRoleHistoryRequestTracking.groovy | 32 ++---
.../model/mock/MockClusterServices.groovy | 5 +
.../appmaster/model/mock/MockFactory.groovy | 4 +-
.../appmaster/model/mock/MockRoleHistory.groovy | 3 +-
10 files changed, 182 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
index 50b5dad..5d52441 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.slider.server.appmaster.state.AbstractClusterServices;
public class ProtobufClusterServices extends AbstractClusterServices {
@@ -27,4 +28,9 @@ public class ProtobufClusterServices extends AbstractClusterServices {
public Resource newResource() {
return Records.newRecord(Resource.class);
}
+
+ @Override
+ public Resource newResource(int memory, int cores) {
+ return Resources.createResource(memory, cores);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
index 27e25f9..eba8c38 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
/**
* Cluster services offered by the YARN infrastructure.
@@ -30,4 +31,9 @@ public abstract class AbstractClusterServices {
*/
public abstract Resource newResource();
+ public abstract Resource newResource(int memory, int cores);
+
+ public Resource normalize(Resource resource, Resource minR, Resource maxR) {
+ return new DefaultResourceCalculator().normalize(resource, minR, maxR);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index f6fe474..1e23bef 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -289,6 +289,8 @@ public class AppState {
* Selector of containers to release; application wide.
*/
private ContainerReleaseSelector containerReleaseSelector;
+ private Resource minResource;
+ private Resource maxResource;
/**
* Create an instance
@@ -309,7 +311,7 @@ public class AppState {
register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount);
}
- private void register(String name, Counter counter) {
+ private void register(String name, Metric counter) {
this.metricsAndMonitoring.getMetrics().register(
MetricRegistry.name(AppState.class, name), counter);
}
@@ -462,6 +464,8 @@ public class AppState {
containerMaxCores = maxCores;
containerMinMemory = minMemory;
containerMaxMemory = maxMemory;
+ minResource = recordFactory.newResource(containerMinMemory, containerMinCores);
+ maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores);
}
public ConfTreeOperations getResourcesSnapshot() {
@@ -556,7 +560,7 @@ public class AppState {
// set up the role history
- roleHistory = new RoleHistory(roleStatusMap.values());
+ roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
roleHistory.register(metricsAndMonitoring);
roleHistory.onStart(binding.fs, binding.historyPath);
// trigger first node update
@@ -762,6 +766,9 @@ public class AppState {
newRoles.add(dynamicRole);
}
}
+ // and fill in all those roles with their requirements
+ buildRoleResourceRequirements();
+
return newRoles;
}
@@ -812,6 +819,17 @@ public class AppState {
}
/**
+ * Build up the requirements of every resource
+ */
+ private void buildRoleResourceRequirements() {
+ roleStatusMap.values();
+ for (RoleStatus role : roleStatusMap.values()) {
+ role.setResourceRequirements(
+ buildResourceRequirements(role, recordFactory.newResource()));
+ }
+ }
+
+ /**
* build up the special master node, which lives
* in the live node set but has a lifecycle bonded to the AM
* @param containerId the AM master
@@ -913,6 +931,12 @@ public class AppState {
}
+ /**
+ * Look up a role in the map
+ * @param name role name
+ * @return the instance
+ * @throws YarnRuntimeException if not found
+ */
public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
ProviderRole providerRole = roles.get(name);
if (providerRole == null) {
@@ -928,7 +952,7 @@ public class AppState {
*/
public synchronized List<RoleInstance> cloneOwnedContainerList() {
Collection<RoleInstance> values = ownedContainers.values();
- return new ArrayList<RoleInstance>(values);
+ return new ArrayList<>(values);
}
/**
@@ -1027,7 +1051,6 @@ public class AppState {
}
}
-
public synchronized List<RoleInstance> getLiveInstancesByContainerIDs(
Collection<String> containerIDs) {
//first, a hashmap of those containerIDs is built up
@@ -1081,7 +1104,6 @@ public class AppState {
return nodes;
}
-
/**
* Build an instance map.
* @return the map of Role name to list of role instances
@@ -1167,38 +1189,18 @@ public class AppState {
/**
- * Set up the resource requirements with all that this role needs,
- * then create the container request itself.
- * @param role role to ask an instance of
- * @param capability a resource to set up
- * @return the request for a new container
- */
- public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
- RoleStatus role,
- Resource capability) {
- buildResourceRequirements(role, capability);
- String labelExpression = role.getLabelExpression();
- //get the role history to select a suitable node, if available
- AMRMClient.ContainerRequest containerRequest =
- createContainerRequest(role, capability);
- return containerRequest;
- }
-
- /**
* Create a container request.
- * Update internal state, such as the role request count
- * This is where role history information will be used for placement decisions -
- * @param labelExpression label expression to satisfy
+ * Update internal state, such as the role request count.
+ * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
+ * This is where role history information will be used for placement decisions.
* @param role role
- * @param resource requirements
* @return the container request to submit
*/
- private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
- Resource resource) {
- AMRMClient.ContainerRequest request;
- request = roleHistory.requestNode(role, resource);
+ private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
incrementRequestCount(role);
- return request;
+ OutstandingRequest request = roleHistory.requestContainerForRole(role);
+ role.setOutstandingAArequest(request);
+ return request.getIssuedRequest();
}
/**
@@ -1266,9 +1268,10 @@ public class AppState {
* cluster specification, including substituing max allowed values
* if the specification asked for it.
* @param role role
- * @param capability capability to set up
+ * @param capability capability to set up. A new one may be created
+ * during normalization
*/
- public void buildResourceRequirements(RoleStatus role, Resource capability) {
+ public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
ConfTreeOperations resources = getResourcesSnapshot();
@@ -1283,6 +1286,7 @@ public class AppState {
DEF_YARN_MEMORY,
containerMaxMemory);
capability.setMemory(ram);
+ return recordFactory.normalize(capability,minResource, maxResource);
}
/**
@@ -1748,7 +1752,7 @@ public class AppState {
public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
throws SliderInternalStateException, TriggerClusterTeardownException {
log.debug("in reviewRequestAndReleaseNodes()");
- List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
+ List<AbstractRMOperation> allOperations = new ArrayList<>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
if (!roleStatus.isExcludeFromFlexing()) {
List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
@@ -1853,6 +1857,7 @@ public class AppState {
long delta;
long expected;
String name = role.getName();
+ boolean isAA = role.isAntiAffinePlacement();
synchronized (role) {
delta = role.getDelta();
expected = role.getDesired();
@@ -1871,38 +1876,43 @@ public class AppState {
}
if (delta > 0) {
- log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
- delta, expected);
// more workers needed than we have -ask for more
- for (int i = 0; i < delta; i++) {
- Resource capability = recordFactory.newResource();
- AMRMClient.ContainerRequest containerAsk =
- buildContainerResourceAndRequest(role, capability);
- log.info("Container ask is {} and label = {}", containerAsk,
- containerAsk.getNodeLabelExpression());
- int askMemory = containerAsk.getCapability().getMemory();
- if (askMemory > this.containerMaxMemory) {
- log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
+ delta, expected);
+
+ // TODO: AA RH to help here by only allowing one request for an AA
+
+ if (isAA) {
+ // build one only if there is none outstanding
+ if (role.getPendingAntiAffineRequests() == 0) {
+ log.info("Starting an anti-affine request sequence");
+ role.incPendingAntiAffineRequests(delta);
+ addContainerRequest(operations, createContainerRequest(role));
+ } else {
+ log.info("Adding {} more anti-affine requests", delta);
+ role.incPendingAntiAffineRequests(delta);
+ }
+ } else {
+
+ for (int i = 0; i < delta; i++) {
+ //get the role history to select a suitable node, if available
+ addContainerRequest(operations, createContainerRequest(role));
}
- operations.add(new ContainerRequestOperation(containerAsk));
}
} else if (delta < 0) {
log.info("{}: Asking for {} fewer node(s) for a total of {}", name,
-delta,
expected);
// reduce the number expected (i.e. subtract the delta)
-
- // then pick some containers to kill
long excess = -delta;
- // how many requests are outstanding
+ // how many requests are outstanding?
long outstandingRequests = role.getRequested();
if (outstandingRequests > 0) {
// outstanding requests.
int toCancel = (int)Math.min(outstandingRequests, excess);
// Delegate to Role History
-
List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel);
log.info("Found {} outstanding requests to cancel", cancellations.size());
operations.addAll(cancellations);
@@ -1922,7 +1932,6 @@ public class AppState {
}
}
-
// after the cancellation there may be no excess
if (excess > 0) {
@@ -1936,7 +1945,7 @@ public class AppState {
log.info("No containers for component {}", roleId);
}
- // cut all release-in-progress nodes
+ // filter out all release-in-progress nodes
ListIterator<RoleInstance> li = containersToRelease.listIterator();
while (li.hasNext()) {
RoleInstance next = li.next();
@@ -1979,6 +1988,17 @@ public class AppState {
return operations;
}
+ private void addContainerRequest(List<AbstractRMOperation> operations,
+ AMRMClient.ContainerRequest containerAsk) {
+ log.info("Container ask is {} and label = {}", containerAsk,
+ containerAsk.getNodeLabelExpression());
+ int askMemory = containerAsk.getCapability().getMemory();
+ if (askMemory > this.containerMaxMemory) {
+ log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ }
+ operations.add(new ContainerRequestOperation(containerAsk));
+ }
+
/**
* Releases a container based on container id
* @param containerId
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index c93c7f5..f8271a6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -73,6 +73,8 @@ public class RoleHistory {
private final List<ProviderRole> providerRoles;
/** the roles in here are shared with App State */
private final Map<Integer, RoleStatus> roleStatusMap = new HashMap<>();
+ private final AbstractClusterServices recordFactory;
+
private long startTime;
/** Time when saved */
@@ -115,9 +117,11 @@ public class RoleHistory {
/**
* Instantiate
* @param roles initial role list
+ * @param recordFactory yarn record factory
* @throws BadConfigException
*/
- public RoleHistory(Collection<RoleStatus> roles) throws BadConfigException {
+ public RoleHistory(Collection<RoleStatus> roles, AbstractClusterServices recordFactory) throws BadConfigException {
+ this.recordFactory = recordFactory;
roleSize = roles.size();
providerRoles = new ArrayList<>(roleSize);
for (RoleStatus role : roles) {
@@ -604,24 +608,27 @@ public class RoleHistory {
*
* @param node node to target or null for "any"
* @param role role to request
- * @return the container priority
+ * @return the request
*/
- public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
+ public synchronized OutstandingRequest requestInstanceOnNode(
NodeInstance node, RoleStatus role, Resource resource) {
OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
- return outstanding.buildContainerRequest(resource, role, now());
+ outstanding.buildContainerRequest(resource, role, now());
+ return outstanding;
}
/**
* Find a node for a role and request an instance on that (or a location-less
* instance)
* @param role role status
- * @param resource resource capabilities
* @return a request ready to go
*/
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource) {
+ public synchronized OutstandingRequest requestContainerForRole(RoleStatus role) {
+
+ Resource resource = recordFactory.newResource();
+ role.copyResourceRequirements(resource);
NodeInstance node = findNodeForNewInstance(role);
+ // TODO AA -what if there are no suitable nodes?
return requestInstanceOnNode(node, role, resource);
}
@@ -992,6 +999,8 @@ public class RoleHistory {
// ask for some placed nodes
requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, remaining));
+ // TODO AA: clear anything here?
+
// build cancellations
for (OutstandingRequest request : requests) {
results.add(request.createCancelOperation());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 157870a..42772c5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -22,6 +22,8 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
@@ -43,16 +45,29 @@ import org.junit.Test
class TestMockAppStateAAPlacement extends BaseMockAppStateTest
implements MockRoles {
- static private final ProviderRole aaRole = MockFactory.PROVIDER_ROLE2
- private static final int roleId = aaRole.id
-/*
+ /**
+ * Patch up a "role2" role to have anti-affinity set
+ */
+ public static final ProviderRole AAROLE = new ProviderRole(
+ MockRoles.ROLE2,
+ 2,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 2,
+ 2,
+ null)
+
@Override
AppStateBindingInfo buildBindingInfo() {
def bindingInfo = super.buildBindingInfo()
- // only have the AA role, to avoid complications/confusion
- bindingInfo.roles = [aaRole]
+ bindingInfo.roles = [
+ MockFactory.PROVIDER_ROLE0,
+ MockFactory.PROVIDER_ROLE1,
+ AAROLE,
+ ]
bindingInfo
- }*/
+ }
+
+ private static final int roleId = AAROLE.id
/**
* Get the single request of a list of operations; includes the check for the size
@@ -74,7 +89,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
@Test
public void testAllocateAANoLabel() throws Throwable {
- def aaRole = lookupRole(aaRole.name)
+ def aaRole = lookupRole(AAROLE.name)
// want two instances, so there will be two iterations
aaRole.desired = 2
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 5609682..d9cfddb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -35,7 +35,6 @@ import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeId
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.*
-import org.junit.Before
import org.junit.Test
/**
@@ -61,13 +60,18 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
String roleName = "test"
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory
Resource resource
- @Before
- public void setupRH() {
- roleHistory.onStart(fs, historyPath)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ roleHistory = appState.roleHistory
roleHistory.insert(nodes)
roleHistory.buildRecentNodeLists();
resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
@@ -77,11 +81,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
@Test
public void testFindAndCreate() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes = request.getNodes()
assert nodes != null
@@ -114,14 +117,13 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
@Test
public void testCreateAndRelease() throws Throwable {
int role = 1
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
//verify it is empty
assert roleHistory.listActiveNodes(role).empty
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
assert request.nodes == null
@@ -170,7 +172,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
// ask for a container and expect to get the recently released one
AMRMClient.ContainerRequest request2 =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes2 = request2.nodes
assert nodes2 != null
@@ -206,11 +208,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
@Test
public void testStartFailed() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -240,10 +241,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
@Test
public void testStartFailedWithoutWarning() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -271,10 +272,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
describe("fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -310,10 +311,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
public void testContainerFailedWithoutWarning() throws Throwable {
describe( "fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -343,10 +344,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
public void testAllocationListPrep() throws Throwable {
describe("test prepareAllocationList")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -375,9 +376,10 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
describe("fail a node")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
- AMRMClient.ContainerRequest request = roleHistory.requestNode(roleStatus, resource);
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest;
String hostname = request.getNodes()[0]
assert age3Active0.hostname == hostname
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 693ea9f..14ac32a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -63,6 +63,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
ProviderRole provRole = new ProviderRole(roleName, 0)
RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
@Override
String getTestName() {
return "TestRoleHistoryAvailableList"
@@ -72,6 +76,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
public void setupNodeMap() {
roleHistory.insert(nodes)
roleHistory.buildRecentNodeLists();
+ roleStatus.setResourceRequirements(Resource.newInstance(1, 1))
}
@Test
@@ -107,8 +112,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// which is translated to a no-location request
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource
- )
+ resource).issuedRequest
assertNull(req.nodes)
@@ -121,7 +125,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// looking for a node should now find one
ni = roleHistory.findNodeForNewInstance(roleStatus)
assert ni == age3Active0
- req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource)
+ req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource).issuedRequest
assert 1 == req.nodes.size()
}
@@ -162,7 +166,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testFindAndRequestNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
assert age3Active0.hostname == req.nodes[0]
List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0)
@@ -171,7 +175,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
@@ -179,7 +183,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
@@ -208,8 +212,8 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testTwoRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -222,9 +226,9 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testThreeRequestsOneUnsatisified() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -251,9 +255,9 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testThreeRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
assertOutstandingPlacedRequests(2)
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
index d27a6bb..dfecc94 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
@@ -27,4 +27,9 @@ class MockClusterServices extends AbstractClusterServices {
Resource newResource() {
return new MockResource()
}
+
+ @Override
+ Resource newResource(int memory, int cores) {
+ return new MockResource(memory, cores)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 0a4a93e..bbd64f1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -59,11 +59,11 @@ class MockFactory implements MockRoles {
2,
1,
ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
- // role 2: longer delay and anti-affinity
+ // role 2: longer delay
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
- PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ PlacementPolicy.NO_DATA_LOCALITY,
2,
2,
ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/aa46b473/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
index 0a68afb..4553e22 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
@@ -35,7 +35,8 @@ class MockRoleHistory extends RoleHistory {
* @throws BadConfigException configuration problem with the role list
*/
MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
- super(providerRoles.collect { new RoleStatus(it) })
+ super(providerRoles.collect { new RoleStatus(it) },
+ new MockClusterServices())
}
}