You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sa...@apache.org on 2014/06/24 17:48:52 UTC
svn commit: r1605115 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-ser...
Author: sandy
Date: Tue Jun 24 15:48:51 2014
New Revision: 1605115
URL: http://svn.apache.org/r1605115
Log:
YARN-2111. In FairScheduler.attemptScheduling, we don't count containers as assigned if they have 0 memory but non-zero cores (Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1605115&r1=1605114&r2=1605115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Jun 24 15:48:51 2014
@@ -255,6 +255,9 @@ Release 2.5.0 - UNRELEASED
YARN-2187. FairScheduler: Disable max-AM-share check by default.
(Robert Kanter via kasha)
+ YARN-2111. In FairScheduler.attemptScheduling, we don't count containers
+ as assigned if they have 0 memory but non-zero cores (Sandy Ryza)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1605115&r1=1605114&r2=1605115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Tue Jun 24 15:48:51 2014
@@ -1033,9 +1033,8 @@ public class FairScheduler extends
int assignedContainers = 0;
while (node.getReservedContainer() == null) {
boolean assignedContainer = false;
- if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- queueMgr.getRootQueue().assignContainer(node),
- Resources.none())) {
+ if (!queueMgr.getRootQueue().assignContainer(node).equals(
+ Resources.none())) {
assignedContainers++;
assignedContainer = true;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1605115&r1=1605114&r2=1605115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Jun 24 15:48:51 2014
@@ -1661,6 +1661,41 @@ public class TestFairScheduler extends F
assertEquals("Incorrect number of containers allocated", 8, app
.getLiveContainers().size());
}
+
+ @Test(timeout = 3000)
+ public void testMaxAssignWithZeroMemoryContainers() throws Exception {
+ conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+
+ ApplicationAttemptId attId =
+ createSchedulingRequest(0, 1, "root.default", "user", 8);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
+
+ // set maxAssign to 2: only 2 containers should be allocated
+ scheduler.maxAssign = 2;
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 2, app
+ .getLiveContainers().size());
+
+ // set maxAssign to -1: all remaining containers should be allocated
+ scheduler.maxAssign = -1;
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 8, app
+ .getLiveContainers().size());
+ }
/**
* Test to verify the behavior of