You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/03/25 21:28:43 UTC
[14/25] incubator-slider git commit: SLIDER-799 test of escalation
through mock
SLIDER-799 test of escalation through mock
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/92e7af6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/92e7af6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/92e7af6c
Branch: refs/heads/develop
Commit: 92e7af6c6c9b874e2e93118d9315ae266c56fb26
Parents: 9e67161
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 18 14:00:33 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 18 14:00:33 2015 +0000
----------------------------------------------------------------------
.../operations/CancelSingleRequest.java | 4 ++
.../slider/server/appmaster/state/AppState.java | 2 +-
.../appmaster/state/OutstandingRequest.java | 16 +++---
.../server/appmaster/state/RoleHistory.java | 4 +-
.../server/appmaster/state/RoleStatus.java | 8 +++
...tRoleHistoryOutstandingRequestTracker.groovy | 55 +++++++++++++++++++-
.../TestRoleHistoryRequestTracking.groovy | 28 +++++-----
.../appmaster/model/mock/MockFactory.groovy | 15 ++++--
8 files changed, 102 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
index 3bb3b09..08eb5bc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -36,6 +36,10 @@ public class CancelSingleRequest extends AbstractRMOperation {
handler.cancelSingleRequest(request);
}
+ public AMRMClient.ContainerRequest getRequest() {
+ return request;
+ }
+
@Override
public String toString() {
return "cancel single request for container at " + request.getPriority().toString();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index a887107..b7ca526 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1203,7 +1203,7 @@ public class AppState {
* @param capability a resource to set up
* @return the request for a new container
*/
- private AMRMClient.ContainerRequest buildContainerResourceAndRequest(
+ public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
RoleStatus role,
Resource capability) {
buildResourceRequirements(role, capability);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 7ad3fbb..799d08e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -140,7 +140,7 @@ public final class OutstandingRequest {
boolean relaxLocality;
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
- boolean usePlacementHistory = role.isStrictPlacement();
+ boolean usePlacementHistory = role.isStrictOrAntiAffinePlacement();
if (!usePlacementHistory) {
// If strict placement does not mandate using placement then check
// that the recent failures on this node is not higher than threshold
@@ -157,7 +157,7 @@ public final class OutstandingRequest {
if (node != null && usePlacementHistory) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = !role.isStrictPlacement();
+ relaxLocality = !role.isStrictOrAntiAffinePlacement();
// tell the node it is in play
node.getOrCreate(roleId);
log.info("Submitting request for container on {}", hosts[0]);
@@ -265,14 +265,16 @@ public final class OutstandingRequest {
result = 31 * result + (hostname != null ? hostname.hashCode() : 0);
return result;
}
-
+
@Override
public String toString() {
- final StringBuilder sb =
- new StringBuilder("OutstandingRequest{");
+ final StringBuilder sb = new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
- sb.append(", node='").append(node).append('\'');
- sb.append(", requestedTime=").append(requestedTimeMillis);
+ sb.append(", node=").append(node);
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
+ sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
+ sb.append(", escalated=").append(escalated);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 7ac5c52..6aefc47 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -835,11 +835,11 @@ public class RoleHistory {
}
/**
- * Get a snapshot of the outstanding request list
+ * Get a snapshot of the outstanding placed request list
* @return a list of the requests outstanding at the time of requesting
*/
@VisibleForTesting
- public synchronized List<OutstandingRequest> getOutstandingRequestList() {
+ public synchronized List<OutstandingRequest> listOutstandingPlacedRequests() {
return outstandingRequests.listOutstandingRequests();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 7ab7dbe..4ee8fad 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -99,6 +99,14 @@ public final class RoleStatus implements Cloneable {
return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT);
}
+ public boolean isAntiAffinePlacement() {
+ return 0 != (getPlacementPolicy() & PlacementPolicy.ANTI_AFFINITY_REQUIRED);
+ }
+
+ public boolean isStrictOrAntiAffinePlacement() {
+ return isStrictPlacement() || isAntiAffinePlacement();
+ }
+
public synchronized int getDesired() {
return desired;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 0143bce..8c79bbf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -18,7 +18,14 @@
package org.apache.slider.server.appmaster.model.history
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockResource
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -70,8 +77,52 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert tracker.resetOutstandingRequests(1).size() == 1
}
+/*
+ @Override
+ AggregateConf buildInstanceDefinition() {
+ def aggregateConf = super.buildInstanceDefinition()
+ def component0 = aggregateConf.resourceOperations.getMandatoryComponent(ROLE0)
+ component0.set(ResourceKeys.COMPONENT_PLACEMENT_POLICY, PlacementPolicy.STRICT)
+ return aggregateConf
+ }
+*/
+
@Test
- public void testEscalation() throws Throwable {
- OutstandingRequest r1 = tracker.newRequest(host1, 0)
+ public void testEscalationOfStrictPlacement() throws Throwable {
+ final def roleStatus = role1Status
+
+
+ ProviderRole role = roleStatus.providerRole
+ assert role.placementPolicy == PlacementPolicy.STRICT;
+ Resource resource = new MockResource()
+
+ appState.buildResourceRequirements(roleStatus, resource)
+
+ // first requst
+ OutstandingRequest r1 = tracker.newRequest(host1, roleStatus.key)
+ final def initialRequest = r1.buildContainerRequest(resource, roleStatus, 0, null)
+ assert r1.issuedRequest != null;
+ assert r1.located
+ assert !r1.escalated
+
+ assert !initialRequest.relaxLocality
+ assert tracker.listOutstandingRequests().size() == 1
+
+ // simulate a few minutes; escalation MUST now be triggered
+ List<AbstractRMOperation> escalations = tracker.escalateOutstandingRequests(180 * 1000)
+
+ assert r1.escalated
+ // two entries
+ assert escalations.size() == 2;
+ final def e1 = escalations[0]
+ assert e1 instanceof CancelSingleRequest
+ CancelSingleRequest cancel = (CancelSingleRequest) e1
+ assert initialRequest == cancel.request
+ final def e2 = escalations[1]
+ assert e2 instanceof ContainerRequestOperation;
+ final def escRequest = (ContainerRequestOperation) e2
+ def req2 = escRequest.request
+ assert req2.relaxLocality
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index d87222d..e593eab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -122,7 +122,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
@@ -130,28 +130,28 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 1
String hostname = requests[0].hostname
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 2, 2)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
@@ -160,22 +160,22 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 2, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
assert !roleHistory.onContainerAllocated(container, 3, 3)
// yet the list is now empty
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -189,17 +189,17 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- assert roleHistory.outstandingRequestList.size() == 2
+ assert roleHistory.listOutstandingPlacedRequests.size() == 2
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
assert roleHistory.onContainerAllocated(container , 3, 1)
- assert roleHistory.outstandingRequestList.size() == 1
+ assert roleHistory.listOutstandingPlacedRequests.size() == 1
container = factory.newContainer(req2, req2.nodes[0])
assert roleHistory.onContainerAllocated(container, 3, 2)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
container = factory.newContainer(req3, "three")
assert !roleHistory.onContainerAllocated(container, 3, 3)
- assert roleHistory.outstandingRequestList.empty
+ assert roleHistory.listOutstandingPlacedRequests.empty
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/92e7af6c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 311c049..37f4021 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -28,6 +28,7 @@ import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
/**
@@ -42,9 +43,15 @@ class MockFactory implements MockRoles {
0)
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
+ 1,
+ PlacementPolicy.STRICT,
+ 3,
1)
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
+ 2,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 4,
2)
int appIdCount;
int attemptIdCount;
@@ -169,12 +176,12 @@ class MockFactory implements MockRoles {
instance.setResources(newConfTree(r1, r2, r3))
return instance
}
-
-
-
+
def roleMap(int count) {
return [
- (ResourceKeys.COMPONENT_INSTANCES):count.toString()
+ (ResourceKeys.COMPONENT_INSTANCES):count.toString(),
+ (ResourceKeys.COMPONENT_PLACEMENT_POLICY):"${PlacementPolicy.STRICT}".toString()
+
]
}