You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by jl...@apache.org on 2014/01/17 18:03:49 UTC
svn commit: r1559184 - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/
hadoop-yarn/hadoop-yarn-server...
Author: jlowe
Date: Fri Jan 17 17:03:49 2014
New Revision: 1559184
URL: http://svn.apache.org/r1559184
Log:
YARN-1592. CapacityScheduler tries to reserve more than a node's total memory on branch-0.23. Contributed by Omkar Vinit Joshi and Thomas Graves
Added:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (with props)
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1559184&r1=1559183&r2=1559184&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Fri Jan 17 17:03:49 2014
@@ -21,6 +21,9 @@ Release 0.23.11 - UNRELEASED
YARN-1180. Update capacity scheduler docs to include types on the configs
(Chen He via jeagles)
+ YARN-1592. CapacityScheduler tries to reserve more than a node's total
+ memory on branch-0.23 (Omkar Vinit Joshi and Thomas Graves via jlowe)
+
Release 0.23.10 - 2013-12-09
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1559184&r1=1559183&r2=1559184&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Fri Jan 17 17:03:49 2014
@@ -119,4 +119,8 @@ public class Resources {
public static Resource max(Resource lhs, Resource rhs) {
return (lhs.getMemory() > rhs.getMemory()) ? lhs : rhs;
}
+
+ public static boolean fitsIn(Resource smaller, Resource bigger) {
+ return smaller.getMemory() <= bigger.getMemory();
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java?rev=1559184&r1=1559183&r2=1559184&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java Fri Jan 17 17:03:49 2014
@@ -52,6 +52,8 @@ public class SchedulerNode {
private RMContainer reservedContainer;
+ private Resource totalResourceCapability;
+
/* set of containers that are allocated containers */
private final Map<ContainerId, RMContainer> launchedContainers =
new HashMap<ContainerId, RMContainer>();
@@ -63,6 +65,7 @@ public class SchedulerNode {
public SchedulerNode(RMNode node) {
this.rmNode = node;
this.availableResource.setMemory(node.getTotalCapability().getMemory());
+ this.totalResourceCapability = Resources.createResource(node.getTotalCapability().getMemory());
}
public RMNode getRMNode() {
@@ -237,5 +240,13 @@ public class SchedulerNode {
public synchronized RMContainer getReservedContainer() {
return reservedContainer;
}
+
+ /**
+ * Get total resources on the node.
+ * @return total resources on the node.
+ */
+ public Resource getTotalResource() {
+ return this.totalResourceCapability;
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1559184&r1=1559183&r2=1559184&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Jan 17 17:03:49 2014
@@ -779,7 +779,7 @@ public class LeafQueue implements CSQueu
LOG.debug("assignContainers: node=" + node.getHostName()
+ " #applications=" + activeApplications.size());
}
-
+
// Check for reserved resources
RMContainer reservedContainer = node.getReservedContainer();
if (reservedContainer != null) {
@@ -1237,9 +1237,16 @@ public class LeafQueue implements CSQueu
+ " request=" + request + " type=" + type);
}
Resource capability = request.getCapability();
-
Resource available = node.getAvailableResource();
+ Resource totalResource = node.getTotalResource();
+ if (!Resources.fitsIn(capability, totalResource)) {
+ LOG.warn("Node : " + node.getNodeID()
+ + " does not have sufficient resource for request : " + request
+ + " node total capability : " + node.getTotalResource());
+ return Resources.none();
+ }
+
assert (available.getMemory() > 0);
// Create the container if necessary
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java?rev=1559184&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java Fri Jan 17 17:03:49 2014
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.junit.Test;
+
+public class TestContainerAllocation {
+
+ private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class);
+
+ private final int GB = 1024;
+
+ @Test(timeout = 3000000)
+ public void testExcessReservationThanNodeManagerCapacity() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ // Register node1
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 2 * GB);
+ MockNM nm2 = rm.registerNode("127.0.0.1:2234", 3 * GB);
+
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+
+ // wait..
+ int waitCount = 20;
+ int size = rm.getRMContext().getRMNodes().size();
+ while ((size = rm.getRMContext().getRMNodes().size()) != 2
+ && waitCount-- > 0) {
+ LOG.info("Waiting for node managers to register : " + size);
+ Thread.sleep(100);
+ }
+ Assert.assertEquals(2, rm.getRMContext().getRMNodes().size());
+ // Submit an application
+ RMApp app1 = rm.submitApp(128);
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ am1.registerAppAttempt();
+
+ LOG.info("sending container requests ");
+ am1.addRequests(new String[] { "*" }, 3 * GB, 1, 1);
+ AMResponse alloc1Response = am1.schedule(); // send the request
+
+ // kick the scheduler
+ nm1.nodeHeartbeat(true);
+ int waitCounter = 20;
+ LOG.info("heartbeating nm1");
+ while (alloc1Response.getAllocatedContainers().size() < 1
+ && waitCounter-- > 0) {
+ LOG.info("Waiting for containers to be created for app 1...");
+ Thread.sleep(500);
+ alloc1Response = am1.schedule();
+ }
+ LOG.info("received container : "
+ + alloc1Response.getAllocatedContainers().size());
+
+ // No container should be allocated.
+ // Internally it should not been reserved.
+ Assert.assertTrue(alloc1Response.getAllocatedContainers().size() == 0);
+
+ LOG.info("heartbeating nm2");
+ waitCounter = 20;
+ nm2.nodeHeartbeat(true);
+ while (alloc1Response.getAllocatedContainers().size() < 1
+ && waitCounter-- > 0) {
+ LOG.info("Waiting for containers to be created for app 1...");
+ Thread.sleep(500);
+ alloc1Response = am1.schedule();
+ }
+ LOG.info("received container : "
+ + alloc1Response.getAllocatedContainers().size());
+ Assert.assertTrue(alloc1Response.getAllocatedContainers().size() == 1);
+
+ rm.stop();
+ }
+}
\ No newline at end of file
Propchange: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
------------------------------------------------------------------------------
svn:eol-style = native