You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tu...@apache.org on 2012/12/21 17:07:11 UTC
svn commit: r1424989 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: tucu
Date: Fri Dec 21 16:07:11 2012
New Revision: 1424989
URL: http://svn.apache.org/viewvc?rev=1424989&view=rev
Log:
YARN-278. Fair scheduler maxRunningApps config causes no apps to make progress. (sandyr via tucu)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1424989&r1=1424988&r2=1424989&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri Dec 21 16:07:11 2012
@@ -146,6 +146,9 @@ Release 2.0.3-alpha - Unreleased
YARN-272. Fair scheduler log messages try to print objects without
overridden toString methods. (sandyr via tucu)
+ YARN-278. Fair scheduler maxRunningApps config causes no apps to make
+ progress. (sandyr via tucu)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1424989&r1=1424988&r2=1424989&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Fri Dec 21 16:07:11 2012
@@ -177,7 +177,9 @@ public class FSLeafQueue extends FSQueue
Collections.sort(appScheds, comparator);
for (AppSchedulable sched: appScheds) {
- return sched.assignContainer(node, reserved);
+ if (sched.getRunnable()) {
+ return sched.assignContainer(node, reserved);
+ }
}
return Resources.none();
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1424989&r1=1424988&r2=1424989&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Dec 21 16:07:11 2012
@@ -161,6 +161,13 @@ public class TestFairScheduler {
scheduler.allocate(id, ask, new ArrayList<ContainerId>());
return id;
}
+
+ private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) {
+ List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+ ResourceRequest request = createResourceRequest(memory, "*", priority, 1);
+ ask.add(request);
+ scheduler.allocate(attId, ask, new ArrayList<ContainerId>());
+ }
// TESTS
@@ -1125,4 +1132,59 @@ public class TestFairScheduler {
assertEquals(0,
scheduler.applications.get(attId2).getCurrentReservation().getMemory());
}
+
+ @Test
+ public void testUserMaxRunningApps() throws Exception {
+ // Set max running apps
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<user name=\"user1\">");
+ out.println("<maxRunningApps>1</maxRunningApps>");
+ out.println("</user>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ // Add a node
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8192));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Request for app 1
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "user1", 1);
+
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+ scheduler.handle(updateEvent);
+
+ // App 1 should be running
+ assertEquals(1, scheduler.applications.get(attId1).getLiveContainers().size());
+
+ ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
+ "user1", 1);
+
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // App 2 should not be running
+ assertEquals(0, scheduler.applications.get(attId2).getLiveContainers().size());
+
+ // Request another container for app 1
+ createSchedulingRequestExistingApplication(1024, 1, attId1);
+
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Request should be fulfilled
+ assertEquals(2, scheduler.applications.get(attId1).getLiveContainers().size());
+ }
}