You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/03/25 21:28:48 UTC
[19/25] incubator-slider git commit: SLIDER-799 SLIDER-817 track
unplaced outstanding requests
SLIDER-799 SLIDER-817 track unplaced outstanding requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43c61fbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43c61fbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43c61fbb
Branch: refs/heads/develop
Commit: 43c61fbba2ee448f2797790aaaf52ddaaf9ac6f5
Parents: 1938323
Author: Steve Loughran <st...@apache.org>
Authored: Tue Mar 24 16:31:02 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Mar 24 16:31:02 2015 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 5 +-
.../slider/server/appmaster/state/AppState.java | 1 +
.../appmaster/state/ContainerAllocation.java | 46 ++++++++++
.../appmaster/state/OutstandingRequest.java | 28 +++++-
.../state/OutstandingRequestTracker.java | 96 +++++++++++++++++---
.../server/appmaster/state/RoleHistory.java | 24 +++--
...tRoleHistoryOutstandingRequestTracker.groovy | 95 +++++++++++++++----
.../TestRoleHistoryRequestTracking.groovy | 32 ++++---
.../appmaster/model/mock/MockPriority.groovy | 43 +++++++++
9 files changed, 314 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index ce52b89..b6cb42b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -995,11 +995,12 @@ public final class SliderUtils {
return "null container";
}
return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s",
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
container.getId(),
container.getNodeId(),
container.getNodeHttpAddress(),
- container.getPriority());
+ container.getPriority(),
+ container.getResource());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 34b0492..20c1792 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1945,6 +1945,7 @@ public class AppState {
ContainerPriority.createPriority(role.getPriority(), true);
Priority p2 =
ContainerPriority.createPriority(role.getPriority(), false);
+ // TODO Delegate to Role History
operations.add(new CancelRequestOperation(p1, p2, toCancel));
role.cancel(toCancel);
excess -= toCancel;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
new file mode 100644
index 0000000..306ffb2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+/**
+ * This is just a tuple of the outcome of a container allocation
+ */
+public class ContainerAllocation {
+
+ /**
+ * What was the outcome of this allocation: placed, escalated, ...
+ */
+ public ContainerAllocationOutcome outcome;
+
+ /**
+ * The outstanding request which originated this.
+ * This will be null if the outcome is {@link ContainerAllocationOutcome#Unallocated}
+ * as it wasn't expected.
+ */
+ public OutstandingRequest origin;
+
+ public ContainerAllocation(ContainerAllocationOutcome outcome,
+ OutstandingRequest origin) {
+ this.outcome = outcome;
+ this.origin = origin;
+ }
+
+ public ContainerAllocation() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 8c320f0..24946af 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -91,6 +91,11 @@ public final class OutstandingRequest {
private boolean mayEscalate;
/**
+ * Priority of request; only valid after the request is built up
+ */
+ private int priority = -1;
+
+ /**
* Create a request
* @param roleId role
* @param node node -can be null
@@ -144,6 +149,10 @@ public final class OutstandingRequest {
return issuedRequest;
}
+ public int getPriority() {
+ return priority;
+ }
+
/**
* Build a container request.
* <p>
@@ -215,6 +224,7 @@ public final class OutstandingRequest {
mayEscalate = false;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
+ priority = pri.getPriority();
issuedRequest = new AMRMClient.ContainerRequest(resource,
hosts,
null,
@@ -236,6 +246,8 @@ public final class OutstandingRequest {
public synchronized AMRMClient.ContainerRequest escalate() {
Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued "+ this);
escalated = true;
+
+ // this is now the priority
Priority pri = ContainerPriority.createPriority(roleId, true);
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
@@ -258,10 +270,13 @@ public final class OutstandingRequest {
/**
* Mark the request as completed (or canceled).
+ * <p>
+ * Current action: if a node is defined, its request count is decremented
*/
public void completed() {
- assert node != null : "null node";
- node.getOrCreate(roleId).requestCompleted();
+ if (node != null) {
+ node.getOrCreate(roleId).requestCompleted();
+ }
}
/**
@@ -277,6 +292,15 @@ public final class OutstandingRequest {
}
/**
+ * Query for the resource requirements matching; always false before a request is issued
+ * @param resource
+ * @return
+ */
+ public synchronized boolean resourceRequirementsMatch(Resource resource) {
+ return issuedRequest != null && issuedRequest.getCapability().equals(resource);
+ }
+
+ /**
* Equality is on hostname and role
* @param o other
* @return true on a match
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index e226a22..05a8052 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -18,8 +18,12 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
@@ -33,6 +37,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
/**
@@ -61,9 +66,15 @@ public class OutstandingRequestTracker {
new HashMap<>();
/**
- * Create a new request for the specific role. If a
- * location is set, the request is added to the list of requests to track.
- * if it isn't, it is not tracked.
+ * List of open requests; no specific details on them.
+ */
+ private List<OutstandingRequest> openRequests = new ArrayList<>();
+
+ /**
+ * Create a new request for the specific role.
+ * <p>
+ * If a location is set, the request is added to {@link #placedRequests}.
+ * If not, it is added to {@link #openRequests}
* <p>
* This does not update the node instance's role's request count
* @param instance node instance to manager
@@ -75,6 +86,8 @@ public class OutstandingRequestTracker {
new OutstandingRequest(role, instance);
if (request.isLocated()) {
placedRequests.put(request, request);
+ } else {
+ openRequests.add(request);
}
return request;
}
@@ -85,7 +98,9 @@ public class OutstandingRequestTracker {
* @param hostname hostname
* @return the request or null if there was no outstanding one in the {@link #placedRequests}
*/
- public synchronized OutstandingRequest lookup(int role, String hostname) {
+ @VisibleForTesting
+ public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) {
+ Preconditions.checkArgument(hostname != null, "null hostname");
return placedRequests.get(new OutstandingRequest(role, hostname));
}
@@ -94,7 +109,8 @@ public class OutstandingRequestTracker {
* @param request matching request to find
* @return the request or null for no match in the {@link #placedRequests}
*/
- public synchronized OutstandingRequest remove(OutstandingRequest request) {
+ @VisibleForTesting
+ public synchronized OutstandingRequest removePlacedRequest(OutstandingRequest request) {
return placedRequests.remove(request);
}
@@ -103,22 +119,62 @@ public class OutstandingRequestTracker {
* from the {@link #placedRequests} structure.
* @param role role index
* @param hostname hostname
+ * @param resource
* @return the allocation outcome
*/
- public synchronized ContainerAllocationOutcome onContainerAllocated(int role, String hostname) {
+ public synchronized ContainerAllocationOutcome onContainerAllocated(int role,
+ String hostname,
+ Container container) {
+ ContainerAllocationOutcome outcome;
OutstandingRequest request =
placedRequests.remove(new OutstandingRequest(role, hostname));
- if (request == null) {
- // not in the list; this is an open placement
- return ContainerAllocationOutcome.Open;
- } else {
+ if (request != null) {
//satisfied request
+ log.info("Found placed request for container: {}", request);
request.completed();
// derive outcome from status of tracked request
- return request.isEscalated()
+ outcome = request.isEscalated()
? ContainerAllocationOutcome.Escalated
: ContainerAllocationOutcome.Placed;
+ } else {
+ // not in the list; this is an open placement
+ // scan through all containers in the open request list
+ request = removeOpenRequest(container);
+ if (request != null) {
+ log.info("Found open request for container: {}", request);
+ request.completed();
+ outcome = ContainerAllocationOutcome.Open;
+ } else {
+ log.warn("Container allocation was not expected :"
+ + SliderUtils.containerToString(container));
+ outcome = ContainerAllocationOutcome.Unallocated;
+ }
+ }
+ return outcome;
+ }
+
+ /**
+ * Find and remove an open request. Determine it by scanning open requests
+ * for one whose priority & resource requirements match that of the container
+ * allocated.
+ * @param container container allocated
+ * @return a request which matches the allocation, or null for "no match"
+ */
+ private OutstandingRequest removeOpenRequest(Container container) {
+ int pri = container.getPriority().getPriority();
+ Resource resource = container.getResource();
+ OutstandingRequest request = null;
+ ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
+ while (openlist.hasNext() && request == null) {
+ OutstandingRequest r = openlist.next();
+ if (r.getPriority() == pri
+ && r.resourceRequirementsMatch(resource)) {
+ // match of priority and resources
+ request = r;
+ openlist.remove();
+ }
}
+ return request;
}
/**
@@ -226,6 +282,13 @@ public class OutstandingRequestTracker {
hosts.add(request.node);
}
}
+ ListIterator<OutstandingRequest> openlist = openRequests.listIterator();
+ while (openlist.hasNext()) {
+ OutstandingRequest next = openlist.next();
+ if (next.roleId == role) {
+ openlist.remove();
+ }
+ }
return hosts;
}
@@ -234,11 +297,20 @@ public class OutstandingRequestTracker {
* are shared
* @return a list of the current outstanding requests
*/
- public synchronized List<OutstandingRequest> listOutstandingRequests() {
+ public synchronized List<OutstandingRequest> listPlacedRequests() {
return new ArrayList<>(placedRequests.values());
}
/**
+ * Get a list of outstanding requests. The list is cloned, but the contents
+ * are shared
+ * @return a list of the current outstanding requests
+ */
+ public synchronized List<OutstandingRequest> listOpenRequests() {
+ return new ArrayList<>(openRequests);
+ }
+
+ /**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 64f9184..0b981b8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -643,12 +643,12 @@ public class RoleHistory {
int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
- LinkedList<NodeInstance> nodeInstances =
- getOrCreateNodesForRoleId(role);
- ContainerAllocationOutcome outcome = outstandingRequests.onContainerAllocated(role, hostname);
+ List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
+ ContainerAllocationOutcome outcome =
+ outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
- // tag nodes as available
+ // clear all the lists, so returning nodes to the available set
List<NodeInstance>
hosts = outstandingRequests.resetOutstandingRequests(role);
if (!hosts.isEmpty()) {
@@ -841,8 +841,18 @@ public class RoleHistory {
* @return a list of the requests outstanding at the time of requesting
*/
@VisibleForTesting
- public synchronized List<OutstandingRequest> listOutstandingPlacedRequests() {
- return outstandingRequests.listOutstandingRequests();
+ public List<OutstandingRequest> listPlacedRequests() {
+ return outstandingRequests.listPlacedRequests();
+ }
+
+
+ /**
+ * Get a snapshot of the outstanding placed request list
+ * @return a list of the requests outstanding at the time of requesting
+ */
+ @VisibleForTesting
+ public List<OutstandingRequest> listOpenRequests() {
+ return outstandingRequests.listOpenRequests();
}
/**
@@ -856,7 +866,6 @@ public class RoleHistory {
return lst;
}
-
/**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
@@ -864,6 +873,5 @@ public class RoleHistory {
public List<AbstractRMOperation> escalateOutstandingRequests() {
long now = now();
return outstandingRequests.escalateOutstandingRequests(now);
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index c30537a..3f41dfd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -23,11 +23,15 @@ import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockContainer
+import org.apache.slider.server.appmaster.model.mock.MockNodeId
+import org.apache.slider.server.appmaster.model.mock.MockPriority
import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
+import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
@@ -49,9 +53,9 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
@Test
public void testAddRetrieveEntry() throws Throwable {
OutstandingRequest request = tracker.newRequest(host1, 0)
- assert tracker.lookup(0, "host1").equals(request)
- assert tracker.remove(request).equals(request)
- assert !tracker.lookup(0, "host1")
+ assert tracker.lookupPlacedRequest(0, "host1").equals(request)
+ assert tracker.removePlacedRequest(request).equals(request)
+ assert !tracker.lookupPlacedRequest(0, "host1")
}
@Test
@@ -59,11 +63,68 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
tracker.newRequest(host1, 0)
tracker.newRequest(host2, 0)
tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1") == ContainerAllocationOutcome.Placed
- assert !tracker.lookup(1, "host1")
- assert tracker.lookup(0, "host1")
+ assert tracker.onContainerAllocated(1, "host1", null) == ContainerAllocationOutcome.Placed
+ assert !tracker.lookupPlacedRequest(1, "host1")
+ assert tracker.lookupPlacedRequest(0, "host1")
}
-
+
+ @Test
+ public void testResetOpenRequests() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ assert !req1.located
+ tracker.newRequest(host1, 0)
+ def openRequests = tracker.listOpenRequests()
+ assert openRequests.size() == 1
+ tracker.resetOutstandingRequests(0)
+ assert tracker.listOpenRequests().empty
+ assert tracker.listPlacedRequests().empty
+ }
+
+ @Test
+ public void testRemoveOpenRequestUnissued() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ assert tracker.listOpenRequests().size() == 1
+ def c1 = factory.newContainer()
+ c1.setPriority(new MockPriority(0))
+
+ def resource = factory.newResource()
+ resource.virtualCores=1
+ resource.memory = 48;
+ c1.setResource(resource)
+ ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1)
+ assert outcome == ContainerAllocationOutcome.Unallocated
+ assert tracker.listOpenRequests().size() == 1
+ }
+
+ @Test
+ public void testIssuedOpenRequest() throws Throwable {
+ def req1 = tracker.newRequest(null, 0)
+ def resource = factory.newResource()
+ resource.virtualCores = 1
+ resource.memory = 48;
+ def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, "")
+ assert tracker.listOpenRequests().size() == 1
+ def c1 = factory.newContainer()
+
+ def nodeId = factory.newNodeId()
+ c1.nodeId = nodeId
+ nodeId.host ="hostname-1"
+
+ def pri = ContainerPriority.buildPriority(0, false)
+ assert pri > 0
+ c1.setPriority(new MockPriority(pri))
+
+ c1.setResource(resource)
+
+ def issued = req1.issuedRequest
+ assert issued.capability == resource
+ assert issued.priority.priority == c1.getPriority().getPriority()
+ assert req1.resourceRequirementsMatch(resource)
+ ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, nodeId.host, c1)
+ assert tracker.listOpenRequests().size() == 0
+ assert outcome == ContainerAllocationOutcome.Open
+ }
+
@Test
public void testResetEntries() throws Throwable {
tracker.newRequest(host1, 0)
@@ -73,14 +134,13 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert canceled.size() == 2
assert canceled.contains(host1)
assert canceled.contains(host2)
- assert tracker.lookup(1, "host1")
- assert !tracker.lookup(0, "host1")
+ assert tracker.lookupPlacedRequest(1, "host1")
+ assert !tracker.lookupPlacedRequest(0, "host1")
canceled = tracker.resetOutstandingRequests(0)
assert canceled.size() == 0
assert tracker.resetOutstandingRequests(1).size() == 1
}
-
@Test
public void testEscalation() throws Throwable {
@@ -92,7 +152,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert outstanding0.located
assert !outstanding0.escalated
assert !initialRequest.relaxLocality
- assert tracker.listOutstandingRequests().size() == 1
+ assert tracker.listPlacedRequests().size() == 1
// second. This one doesn't get launched. This is to verify that the escalation
// process skips entries which are in the list but have not been issued.
@@ -137,7 +197,7 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
assert escalations3.size() == 2
assert outstanding2.escalated
- // finally add a strict entry to th emix
+ // finally add a strict entry to the mix
def (res3, outstanding3) = newRequest(role1Status)
final ProviderRole providerRole1 = role1Status.providerRole
assert providerRole1.placementPolicy == PlacementPolicy.STRICT
@@ -156,7 +216,8 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
entry.containerCompleted(false)
entry.containerCompleted(false)
assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ final AMRMClient.ContainerRequest initialRequest =
+ outstanding0.buildContainerRequest(res0, role0Status, 0, null)
assert initialRequest.relaxLocality
assert initialRequest.nodes == null
}
@@ -171,16 +232,16 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
entry.containerCompleted(false)
entry.containerCompleted(false)
assert entry.failedRecently == 3
- final AMRMClient.ContainerRequest initialRequest = outstanding0.buildContainerRequest(res0,
- roleStatus, 0, null)
+ final AMRMClient.ContainerRequest initialRequest =
+ outstanding0.buildContainerRequest(res0, roleStatus, 0, null)
assert !initialRequest.relaxLocality
assert initialRequest.nodes[0] == host1.hostname
}
/**
* Create a new request (always against host1)
- * @param r
- * @return
+ * @param r role status
+ * @return (resource, oustanding-request)
*/
public def newRequest(RoleStatus r) {
final Resource res2 = new MockResource()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index bab6233..34a0a4d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.providers.ProviderRole
@@ -123,7 +124,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
}
@@ -131,44 +132,44 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Test
public void testCompletedRequestDropsNode() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
assert age3Active0.hostname == hostname
assert hostname == req.nodes[0]
MockContainer container = factory.newContainer(req, hostname)
assertOnContainerAllocated(container, 2, 1)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
- public void assertOnContainerAllocated(MockContainer c1, int p1, int p2) {
+ public void assertOnContainerAllocated(Container c1, int p1, int p2) {
assert ContainerAllocationOutcome.Open != roleHistory.onContainerAllocated(c1, p1, p2)
}
- public void assertOnContainerAllocationOpen(MockContainer c1, int p1, int p2) {
+ public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
assert ContainerAllocationOutcome.Open == roleHistory.onContainerAllocated(c1, p1, p2)
}
- def assertNoOutstandingRequests() {
- assert roleHistory.listOutstandingPlacedRequests().empty
+ def assertNoOutstandingPlacedRequests() {
+ assert roleHistory.listPlacedRequests().empty
}
public void assertOutstandingPlacedRequests(int i) {
- assert roleHistory.listOutstandingPlacedRequests().size() == i
+ assert roleHistory.listPlacedRequests().size() == i
}
@Test
public void testTwoRequests() throws Throwable {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assertOnContainerAllocated(container, 2, 1)
assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
assertOnContainerAllocated(container, 2, 2)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
@Test
@@ -176,7 +177,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
- List<OutstandingRequest> requests = roleHistory.listOutstandingPlacedRequests()
+ List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
assertOnContainerAllocated(container, 2, 1)
@@ -189,9 +190,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
// the final allocation will trigger a cleanup
container = factory.newContainer(req2, "four")
// no node dropped
- assertOnContainerAllocationOpen(container, 3, 3)
+ assert ContainerAllocationOutcome.Unallocated == roleHistory.onContainerAllocated(container, 3, 3)
// yet the list is now empty
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
+ roleHistory.listOpenRequests().empty
// and the remainder goes onto the available list
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
@@ -211,10 +213,10 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
assertOutstandingPlacedRequests(1)
container = factory.newContainer(req2, req2.nodes[0])
assertOnContainerAllocated(container, 3, 2)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
container = factory.newContainer(req3, "three")
assertOnContainerAllocationOpen(container, 3, 3)
- assertNoOutstandingRequests()
+ assertNoOutstandingPlacedRequests()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43c61fbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
new file mode 100644
index 0000000..9737973
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockPriority.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.Priority
+
+class MockPriority extends Priority {
+
+ private int priority;
+
+ MockPriority(int priority) {
+ this.priority = priority
+ }
+
+ MockPriority() {
+ }
+
+ @Override
+ int getPriority() {
+ return priority
+ }
+
+ @Override
+ void setPriority(int priority) {
+
+ }
+}