You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2018/01/24 22:48:17 UTC
[16/42] hadoop git commit: YARN-7774. Miscellaneous fixes to the
PlacementProcessor. (asuresh)
YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d08342c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d08342c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d08342c5
Branch: refs/heads/YARN-6592
Commit: d08342c55c41970ab6a9c3e1d4ff669efce214d1
Parents: adb8311
Author: Arun Suresh <as...@apache.org>
Authored: Thu Jan 18 11:01:36 2018 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 24 14:20:34 2018 -0800
----------------------------------------------------------------------
.../scheduler/SchedulerNode.java | 14 ++++
.../scheduler/capacity/CapacityScheduler.java | 4 +
.../constraint/PlacementConstraintsUtil.java | 5 +-
.../constraint/algorithm/CircularIterator.java | 86 ++++++++++++++++++++
.../algorithm/DefaultPlacementAlgorithm.java | 50 ++++++++++--
.../constraint/processor/BatchedRequests.java | 8 ++
.../SingleConstraintAppPlacementAllocator.java | 2 +-
.../yarn/server/resourcemanager/MockAM.java | 4 +-
.../constraint/TestPlacementProcessor.java | 24 +++---
.../algorithm/TestCircularIterator.java | 84 +++++++++++++++++++
...stSingleConstraintAppPlacementAllocator.java | 28 +++----
11 files changed, 270 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 05dbf1e..253eb24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -453,6 +453,20 @@ public abstract class SchedulerNode {
return this.nodeUtilization;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof SchedulerNode)) return false;
+
+ SchedulerNode that = (SchedulerNode) o;
+
+ return getNodeID().equals(that.getNodeID());
+ }
+
+ @Override
+ public int hashCode() {
+ return getNodeID().hashCode();
+ }
private static class ContainerInfo {
private final RMContainer container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 8b39aa6..b1c7298 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2550,6 +2550,10 @@ public class CapacityScheduler extends
" but only 1 will be attempted !!");
}
if (!appAttempt.isStopped()) {
+ Resource resource =
+ schedulingRequest.getResourceSizing().getResources();
+ schedulingRequest.getResourceSizing().setResources(
+ getNormalizedResource(resource));
ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
resourceCommitRequest = createResourceCommitRequest(
appAttempt, schedulingRequest, schedulerNode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 24c5a5e..ff5cb67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -99,14 +99,11 @@ public final class PlacementConstraintsUtil {
targetApplicationId, te.getTargetValues(), Long::min);
}
}
- // Make sure Anti-affinity satisfies hard upper limit
- maxScopeCardinality = desiredMaxCardinality == 0 ? maxScopeCardinality - 1
- : maxScopeCardinality;
return (desiredMinCardinality <= 0
|| minScopeCardinality >= desiredMinCardinality) && (
desiredMaxCardinality == Integer.MAX_VALUE
- || maxScopeCardinality < desiredMaxCardinality);
+ || maxScopeCardinality <= desiredMaxCardinality);
}
private static boolean canSatisfyNodePartitionConstraintExpresssion(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/CircularIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/CircularIterator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/CircularIterator.java
new file mode 100644
index 0000000..bf9503b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/CircularIterator.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
+
+import java.util.Iterator;
+
+/**
+ * Iterator that can take current state of an existing iterator
+ * and circularly iterate to that point.
+ */
+class CircularIterator<T> {
+ private Iterator<T> iterator = null;
+ private final Iterable<T> iterable;
+
+ private T startElem = null;
+ private T nextElem = null;
+
+ // if not null, This overrides the starting Element.
+ private T firstElem = null;
+
+ // Can't handle empty or null lists.
+ CircularIterator(T first, Iterator<T> iter,
+ Iterable<T> iterable) {
+ this.firstElem = first;
+ this.iterable = iterable;
+ if (!iter.hasNext()) {
+ this.iterator = this.iterable.iterator();
+ } else {
+ this.iterator = iter;
+ }
+ this.startElem = this.iterator.next();
+ this.nextElem = this.startElem;
+ }
+
+ boolean hasNext() {
+ if (this.nextElem != null || this.firstElem != null) {
+ return true;
+ } else {
+ if (this.iterator.hasNext()) {
+ T next = this.iterator.next();
+ if (this.startElem.equals(next)) {
+ return false;
+ } else {
+ this.nextElem = next;
+ return true;
+ }
+ } else {
+ this.iterator = this.iterable.iterator();
+ this.nextElem = this.iterator.next();
+ if (this.startElem.equals(this.nextElem)) {
+ return false;
+ }
+ return true;
+ }
+ }
+ }
+
+ T next() {
+ T retVal;
+ if (this.firstElem != null) {
+ retVal = this.firstElem;
+ this.firstElem = null;
+ } else if (this.nextElem != null) {
+ retVal = this.nextElem;
+ this.nextElem = null;
+ } else {
+ retVal = this.iterator.next();
+ }
+ return retVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index eb3fe88..a0749f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -49,6 +50,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultPlacementAlgorithm.class);
+ // Number of times to re-attempt placing a single scheduling request.
+ private static final int RE_ATTEMPT_COUNT = 2;
+
private AllocationTagsManager tagsManager;
private PlacementConstraintManager constraintManager;
private NodeCandidateSelector nodeSelector;
@@ -85,16 +89,50 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
new ConstraintPlacementAlgorithmOutput(requests.getApplicationId());
List<SchedulerNode> allNodes = nodeSelector.selectNodes(null);
+ List<SchedulingRequest> rejectedRequests = new ArrayList<>();
+ int rePlacementCount = RE_ATTEMPT_COUNT;
+ while (rePlacementCount > 0) {
+ doPlacement(requests, resp, allNodes, rejectedRequests);
+ if (rejectedRequests.size() == 0 || rePlacementCount == 1) {
+ break;
+ }
+ requests = new BatchedRequests(requests.getIteratorType(),
+ requests.getApplicationId(), rejectedRequests,
+ requests.getPlacementAttempt());
+ rejectedRequests = new ArrayList<>();
+ rePlacementCount--;
+ }
+
+ resp.getRejectedRequests().addAll(rejectedRequests);
+ collector.collect(resp);
+ // Clean current temp-container tags
+ this.tagsManager.cleanTempContainers(requests.getApplicationId());
+ }
+
+ private void doPlacement(BatchedRequests requests,
+ ConstraintPlacementAlgorithmOutput resp,
+ List<SchedulerNode> allNodes,
+ List<SchedulingRequest> rejectedRequests) {
Iterator<SchedulingRequest> requestIterator = requests.iterator();
+ Iterator<SchedulerNode> nIter = allNodes.iterator();
+ SchedulerNode lastSatisfiedNode = null;
while (requestIterator.hasNext()) {
+ if (allNodes.isEmpty()) {
+ LOG.warn("No nodes available for placement at the moment !!");
+ break;
+ }
SchedulingRequest schedulingRequest = requestIterator.next();
- Iterator<SchedulerNode> nodeIter = allNodes.iterator();
+ CircularIterator<SchedulerNode> nodeIter =
+ new CircularIterator(lastSatisfiedNode, nIter, allNodes);
int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
while (nodeIter.hasNext() && numAllocs > 0) {
SchedulerNode node = nodeIter.next();
try {
- if (attemptPlacementOnNode(requests.getApplicationId(),
- schedulingRequest, node)) {
+ String tag = schedulingRequest.getAllocationTags() == null ? "" :
+ schedulingRequest.getAllocationTags().iterator().next();
+ if (!requests.getBlacklist(tag).contains(node.getNodeID()) &&
+ attemptPlacementOnNode(
+ requests.getApplicationId(), schedulingRequest, node)) {
schedulingRequest.getResourceSizing()
.setNumAllocations(--numAllocs);
PlacedSchedulingRequest placedReq =
@@ -108,6 +146,7 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
this.tagsManager.addTempContainer(node.getNodeID(),
requests.getApplicationId(),
schedulingRequest.getAllocationTags());
+ lastSatisfiedNode = node;
}
} catch (InvalidAllocationTagsQueryException e) {
LOG.warn("Got exception from TagManager !", e);
@@ -117,9 +156,6 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
// Add all requests whose numAllocations still > 0 to rejected list.
requests.getSchedulingRequests().stream()
.filter(sReq -> sReq.getResourceSizing().getNumAllocations() > 0)
- .forEach(rejReq -> resp.getRejectedRequests().add(rejReq));
- collector.collect(resp);
- // Clean current temp-container tags
- this.tagsManager.cleanTempContainers(requests.getApplicationId());
+ .forEach(rejReq -> rejectedRequests.add(rejReq));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
index 8b04860..8e39b63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
@@ -133,4 +133,12 @@ public class BatchedRequests
public Set<NodeId> getBlacklist(String tag) {
return blacklist.getOrDefault(tag, Collections.EMPTY_SET);
}
+
+ /**
+ * Get Iterator type.
+ * @return Iterator type.
+ */
+ public IteratorType getIteratorType() {
+ return iteratorType;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
index f8f758c..dd30b61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
@@ -263,7 +263,7 @@ public class SingleConstraintAppPlacementAllocator<N extends SchedulerNode>
}
if (singleConstraint.getMinCardinality() != 0
- || singleConstraint.getMaxCardinality() != 1) {
+ || singleConstraint.getMaxCardinality() != 0) {
throwExceptionWithMetaInfo(
"Only support anti-affinity, which is: minCardinality=0, "
+ "maxCardinality=1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index 9fa2c40..2ed201c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -311,7 +311,7 @@ public class MockAM {
.allocationRequestId(allocationId).priority(priority)
.allocationTags(allocationTags).placementConstraintExpression(
PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(targetTags)).build())
.resourceSizing(resourceSizing).build()), null);
@@ -325,7 +325,7 @@ public class MockAM {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(allocationId).priority(priority)
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp(tags),
PlacementConstraints.PlacementTargets
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
index c260fe0..65daeb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
@@ -153,13 +153,13 @@ public class TestPlacementProcessor {
@Test(timeout = 300000)
public void testCardinalityPlacement() throws Exception {
HashMap<NodeId, MockNM> nodes = new HashMap<>();
- MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm1 = new MockNM("h1:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm1.getNodeId(), nm1);
- MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm2 = new MockNM("h2:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm2.getNodeId(), nm2);
- MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm3 = new MockNM("h3:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm3.getNodeId(), nm3);
- MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm4 = new MockNM("h4:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm4.getNodeId(), nm4);
nm1.registerNode();
nm2.registerNode();
@@ -171,7 +171,7 @@ public class TestPlacementProcessor {
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
Collections.singletonMap(Collections.singleton("foo"),
PlacementConstraints.build(PlacementConstraints
- .targetCardinality(NODE, 0, 4, allocationTag("foo")))));
+ .targetCardinality(NODE, 0, 3, allocationTag("foo")))));
am1.addSchedulingRequest(
Arrays.asList(schedulingRequest(1, 1, 1, 512, "foo"),
schedulingRequest(1, 2, 1, 512, "foo"),
@@ -201,13 +201,13 @@ public class TestPlacementProcessor {
@Test(timeout = 300000)
public void testAffinityPlacement() throws Exception {
HashMap<NodeId, MockNM> nodes = new HashMap<>();
- MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm1 = new MockNM("h1:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm1.getNodeId(), nm1);
- MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm2 = new MockNM("h2:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm2.getNodeId(), nm2);
- MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm3 = new MockNM("h3:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm3.getNodeId(), nm3);
- MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ MockNM nm4 = new MockNM("h4:1234", 8192, rm.getResourceTrackerService());
nodes.put(nm4.getNodeId(), nm4);
nm1.registerNode();
nm2.registerNode();
@@ -267,7 +267,7 @@ public class TestPlacementProcessor {
PlacementConstraints.build(targetIn(NODE, allocationTag("bar"))));
// Containers with allocationTag 'foo' should not exceed 2 per NODE
constraintMap.put(Collections.singleton("foo"), PlacementConstraints
- .build(targetCardinality(NODE, 0, 2, allocationTag("foo"))));
+ .build(targetCardinality(NODE, 0, 1, allocationTag("foo"))));
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, constraintMap);
am1.addSchedulingRequest(
Arrays.asList(schedulingRequest(1, 1, 1, 512, "bar"),
@@ -513,7 +513,8 @@ public class TestPlacementProcessor {
private static void waitForContainerAllocation(Collection<MockNM> nodes,
MockAM am, List<Container> allocatedContainers, int containerNum)
throws Exception {
- while (allocatedContainers.size() < containerNum) {
+ int attemptCount = 10;
+ while (allocatedContainers.size() < containerNum && attemptCount > 0) {
for (MockNM node : nodes) {
node.nodeHeartbeat(true);
}
@@ -522,6 +523,7 @@ public class TestPlacementProcessor {
sleep(1000);
AllocateResponse allocResponse = am.schedule();
allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ attemptCount--;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestCircularIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestCircularIterator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestCircularIterator.java
new file mode 100644
index 0000000..5ce76b0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestCircularIterator.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Simple test case to test the Circular Iterator.
+ */
+public class TestCircularIterator {
+
+ @Test
+ public void testIteration() throws Exception {
+ List<String> list = Arrays.asList("a", "b", "c", "d");
+ CircularIterator<String> ci =
+ new CircularIterator<>(null, list.iterator(), list);
+ StringBuffer sb = new StringBuffer("");
+ while (ci.hasNext()) {
+ sb.append(ci.next());
+ }
+ Assert.assertEquals("abcd", sb.toString());
+
+ Iterator<String> lIter = list.iterator();
+ lIter.next();
+ lIter.next();
+ sb = new StringBuffer("");
+ ci = new CircularIterator<>(null, lIter, list);
+ while (ci.hasNext()) {
+ sb.append(ci.next());
+ }
+ Assert.assertEquals("cdab", sb.toString());
+
+ lIter = list.iterator();
+ lIter.next();
+ lIter.next();
+ lIter.next();
+ sb = new StringBuffer("");
+ ci = new CircularIterator<>("x", lIter, list);
+ while (ci.hasNext()) {
+ sb.append(ci.next());
+ }
+ Assert.assertEquals("xdabc", sb.toString());
+
+ list = Arrays.asList("a");
+ lIter = list.iterator();
+ lIter.next();
+ sb = new StringBuffer("");
+ ci = new CircularIterator<>("y", lIter, list);
+ while (ci.hasNext()) {
+ sb.append(ci.next());
+ }
+ Assert.assertEquals("ya", sb.toString());
+
+ try {
+ list = new ArrayList<>();
+ lIter = list.iterator();
+ new CircularIterator<>("y", lIter, list);
+ Assert.fail("Should fail..");
+ } catch (Exception e) {
+ // foo bar
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08342c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
index 479d2c1..3485ea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java
@@ -118,7 +118,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -134,7 +134,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x"))
@@ -150,7 +150,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.resourceSizing(
@@ -165,7 +165,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.resourceSizing(
@@ -181,7 +181,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer")).build())
.build(), true);
@@ -191,7 +191,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1).build())
+ .targetNotIn(PlacementConstraints.NODE).build())
.build(), true);
// Invalid (with multiple allocation tags expression specified)
@@ -199,7 +199,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
PlacementConstraints.PlacementTargets
@@ -214,7 +214,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper"),
PlacementConstraints.PlacementTargets
@@ -255,7 +255,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.RACK, 0, 1,
+ .targetNotIn(PlacementConstraints.RACK,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -268,7 +268,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -284,7 +284,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -330,7 +330,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -350,7 +350,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition(""))
@@ -372,7 +372,7 @@ public class TestSingleConstraintAppPlacementAllocator {
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED))
.allocationRequestId(10L).priority(Priority.newInstance(1))
.placementConstraintExpression(PlacementConstraints
- .targetCardinality(PlacementConstraints.NODE, 0, 1,
+ .targetNotIn(PlacementConstraints.NODE,
PlacementConstraints.PlacementTargets
.allocationTagToIntraApp("mapper", "reducer"),
PlacementConstraints.PlacementTargets.nodePartition("x"))
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org