You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/23 18:20:14 UTC
[02/50] incubator-slider git commit: SLIDER-963 Write mock/unit tests
for AA placement -initial test; enough for the code to be written against
SLIDER-963 Write mock/unit tests for AA placement -initial test; enough for the code to be written against
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f2786ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f2786ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f2786ca
Branch: refs/heads/develop
Commit: 8f2786ca53cdac06abb851ca4fd49944e59019bf
Parents: 0fce42f
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 6 15:05:23 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 6 15:05:23 2015 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 52 +++++-----
.../appstate/TestMockAppStateAAPlacement.groovy | 104 +++++++++++++++----
.../model/mock/BaseMockAppStateTest.groovy | 3 +-
.../appmaster/model/mock/MockNodeReport.groovy | 40 +++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 15 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 7 ++
6 files changed, 173 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 946d45f..29d5cde 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -529,8 +529,7 @@ public class AppState {
// this is a new value
log.info("Adding role {}", name);
MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole =
- createDynamicProviderRole(name, resComponent);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
roleList.add(dynamicRole);
}
@@ -546,11 +545,11 @@ public class AppState {
InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ CONTAINER_FAILURE_THRESHOLD,
+ DEFAULT_CONTAINER_FAILURE_THRESHOLD);
nodeFailureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD);
+ NODE_FAILURE_THRESHOLD,
+ DEFAULT_NODE_FAILURE_THRESHOLD);
initClusterStatus();
@@ -606,22 +605,21 @@ public class AppState {
* @return a new provider role
* @throws BadConfigException bad configuration
*/
- public ProviderRole createDynamicProviderRole(String name,
- MapOperations component) throws
- BadConfigException {
- String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int priority = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PRIORITY,
- priOpt, 0, 1, -1);
- String placementOpt = component.getOption(
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ public ProviderRole createDynamicProviderRole(String name, MapOperations component)
+ throws BadConfigException {
+ String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY);
+ int priority = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1);
+
+ String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.DEFAULT));
- int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
- placementOpt, 0, 0, -1);
- int placementTimeout =
- component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
+ int placement = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1);
+
+ int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY,
+ DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
ProviderRole newRole = new ProviderRole(name,
priority,
placement,
@@ -662,7 +660,7 @@ public class AppState {
instanceDefinition.getResourceOperations();
if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) {
resources.setComponentOpt(
- SliderKeys.COMPONENT_AM, ResourceKeys.COMPONENT_INSTANCES, "1");
+ SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1");
}
@@ -780,7 +778,7 @@ public class AppState {
private int getDesiredInstanceCount(ConfTreeOperations resources,
String role) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
log.error("Role {} has negative desired instances : {}", role,
@@ -1271,7 +1269,7 @@ public class AppState {
String val = resources.getComponentOpt(name, option,
Integer.toString(defVal));
Integer intVal;
- if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ if (YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
} else {
intVal = Integer.decode(val);
@@ -1679,7 +1677,7 @@ public class AppState {
String rolename = role.getName();
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
- cd.setRoleOpt(rolename, ResourceKeys.COMPONENT_INSTANCES,
+ cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
@@ -1813,7 +1811,7 @@ public class AppState {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
+ CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
@@ -1827,7 +1825,7 @@ public class AppState {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleName,
- ResourceKeys.NODE_FAILURE_THRESHOLD,
+ NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 6168146..810affc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -21,21 +21,20 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.api.records.NodeId
import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
-import org.apache.slider.server.appmaster.state.RoleHistoryUtils
+import org.apache.slider.server.appmaster.state.NodeMap
import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
/**
* Test Anti-affine placement
*/
@@ -44,17 +43,79 @@ import static org.apache.slider.server.appmaster.state.ContainerPriority.extract
class TestMockAppStateAAPlacement extends BaseMockAppStateTest
implements MockRoles {
-// @Test
- public void testAllocateAA() throws Throwable {
+ static private final ProviderRole aaRole = MockFactory.PROVIDER_ROLE2
+ private static final int roleId = aaRole.id
+/*
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ // only have the AA role, to avoid complications/confusion
+ bindingInfo.roles = [aaRole]
+ bindingInfo
+ }*/
+
+ /**
+ * Get the container request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof ContainerRequestOperation
+ ((ContainerRequestOperation) op).request
+ }
- def aaRole = role2Status
+ /**
+ * Get the cancel request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof CancelSingleRequest
+ ((CancelSingleRequest) op).request
+ }
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first ContainerRequestOperation
+ */
+ public AMRMClient.ContainerRequest getSingleRequest(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getRequest(ops, 0)
+ }
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first operation
+ */
+ public AMRMClient.ContainerRequest getSingleCancel(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getCancel(ops, 0)
+ }
+
+ @Test
+ public void testVerifyNodeMap() throws Throwable {
+
+ def nodemap = appState.roleHistory.cloneNodemap()
+ assert nodemap.size() > 0
+ }
+
+ @Test
+ public void testAllocateAANoLabel() throws Throwable {
+
+ def aaRole = lookupRole(aaRole.name)
+
+ // want two instances, so there will be two iterations
aaRole.desired = 2
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert 1 == ops.size()
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request = operation.request
+ AMRMClient.ContainerRequest request = getSingleRequest(ops)
assert request.relaxLocality
assert request.nodes == null
assert request.racks == null
@@ -68,18 +129,22 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
// notify the container ane expect
List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releaseOperations = []
- appState.onContainersAllocated([allocated], assignments, releaseOperations)
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated([allocated], assignments, operations)
- // verify the release matches the allocation
- assert releaseOperations.size() == 1
- CancelSingleRequest cancelOp = releaseOperations[0] as CancelSingleRequest;
- assert cancelOp.request.capability.equals(allocated.resource)
- // now the assignment
+ // assignment
assert assignments.size() == 1
+ // verify the release matches the allocation
+ assert operations.size() == 2
+ assert getCancel(operations, 0).capability.equals(allocated.resource)
+
// we also expect a new allocation request to have been issued
- //
+
+ def req2 = getRequest(operations, 1)
+ // now the nodes should be a list
+ Container allocated2 = engine.allocateContainer(req2)
+
ContainerAssignment assigned = assignments[0]
Container container = assigned.container
@@ -89,6 +154,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
assert appState.onNodeManagerContainerStarted(container.id)
ops = appState.reviewRequestAndReleaseNodes()
assert ops.size() == 0
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 40d7fd7..44d35be 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.common.tools.SliderUtils
@@ -88,7 +89,6 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
applicationId: applicationId,
attemptId: 1)
- fs = HadoopFS.get(new URI("file:///"), conf)
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
@@ -108,6 +108,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
binding.roles = factory.ROLES
binding.fs = fs
binding.historyPath = historyPath
+ binding.nodeReports = engine.nodeReports as List<NodeReport>
binding
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
new file mode 100644
index 0000000..1c7a816
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.Resource
+
+/**
+ * Node report for testing
+ */
+class MockNodeReport extends NodeReport {
+ NodeId nodeId;
+ NodeState nodeState;
+ String httpAddress;
+ String rackName;
+ Resource used;
+ Resource capability;
+ int numContainers;
+ String healthReport;
+ long lastHealthReportTime;
+ Set<String> nodeLabels;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 99a9213..265a796 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeState
/**
* Models the cluster itself: a set of mock cluster nodes.
@@ -143,7 +144,14 @@ public class MockYarnCluster {
}
return total;
}
-
+
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ nodes.collect { MockYarnClusterNode n -> n.nodeReport }
+ }
/**
* Model cluster nodes on the simpler "slot" model than the YARN-era
@@ -159,6 +167,7 @@ public class MockYarnCluster {
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
+ public MockNodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -171,6 +180,10 @@ public class MockYarnCluster {
MockContainerId mci = new MockContainerId(containerId: cid)
containers[i] = new MockYarnClusterContainer(mci)
}
+
+ nodeReport = new MockNodeReport()
+ nodeReport.nodeId = nodeId
+ nodeReport.nodeState = NodeState.RUNNING
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f2786ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 5860c6b..965219d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -157,4 +157,11 @@ class MockYarnEngine {
}
}
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ cluster.nodeReports
+ }
}
\ No newline at end of file