You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2012/11/30 13:03:28 UTC
svn commit: r1415592 [2/2] - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-server/ha...
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1415592&r1=1415591&r2=1415592&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Nov 30 12:03:25 2012
@@ -31,6 +31,10 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import javax.xml.parsers.ParserConfigurationException;
+
+import junit.framework.Assert;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.Clock;
@@ -61,6 +65,7 @@ import org.apache.hadoop.yarn.server.res
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.xml.sax.SAXException;
public class TestFairScheduler {
@@ -195,15 +200,64 @@ public class TestFairScheduler {
scheduler.update();
- Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+ Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
- for (FSQueue p : queues) {
- if (p.getName() != "default") {
- assertEquals(5120, p.getQueueSchedulable().getFairShare().getMemory());
+ for (FSLeafQueue p : queues) {
+ if (!p.getName().equals("root.default")) {
+ assertEquals(5120, p.getFairShare().getMemory());
}
}
}
+
+ @Test
+ public void testSimpleHierarchicalFairShareCalculation() {
+ // Add one big node (only care about aggregate capacity)
+ int capacity = 10 * 24;
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Have two queues which want entire cluster capacity
+ createSchedulingRequest(10 * 1024, "queue1", "user1");
+ createSchedulingRequest(10 * 1024, "parent.queue2", "user1");
+ createSchedulingRequest(10 * 1024, "parent.queue3", "user1");
+
+ scheduler.update();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
+ assertEquals(4, queues.size());
+
+ FSLeafQueue queue1 = queueManager.getLeafQueue("queue1");
+ FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2");
+ FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3");
+ assertEquals(capacity / 2, queue1.getFairShare().getMemory());
+ assertEquals(capacity / 4, queue2.getFairShare().getMemory());
+ assertEquals(capacity / 4, queue3.getFairShare().getMemory());
+ }
+
+ @Test
+ public void testHierarchicalQueuesSimilarParents() {
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child");
+ Assert.assertEquals(2, queueManager.getLeafQueues().size());
+ Assert.assertNotNull(leafQueue);
+ Assert.assertEquals("root.parent.child", leafQueue.getName());
+
+ FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent");
+ Assert.assertNull(leafQueue2);
+ Assert.assertEquals(2, queueManager.getLeafQueues().size());
+
+ FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild");
+ Assert.assertNull(leafQueue3);
+ Assert.assertEquals(2, queueManager.getLeafQueues().size());
+
+ FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister");
+ Assert.assertNotNull(leafQueue4);
+ Assert.assertEquals("root.parent.sister", leafQueue4.getName());
+ Assert.assertEquals(3, queueManager.getLeafQueues().size());
+ }
@Test
public void testSimpleContainerAllocation() {
@@ -228,14 +282,14 @@ public class TestFairScheduler {
// Asked for less than min_allocation.
assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
scheduler.getQueueManager().getQueue("queue1").
- getQueueSchedulable().getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
scheduler.handle(updateEvent2);
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getQueueSchedulable().getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
}
@Test
@@ -254,7 +308,7 @@ public class TestFairScheduler {
// Make sure queue 1 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getQueueSchedulable().getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
// Now queue 2 requests likewise
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -263,7 +317,7 @@ public class TestFairScheduler {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getQueueSchedulable().getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory());
// Now another node checks in with capacity
@@ -276,7 +330,7 @@ public class TestFairScheduler {
// Make sure this goes to queue 2
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getQueueSchedulable().getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
// The old reservation should still be there...
assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory());
@@ -294,17 +348,22 @@ public class TestFairScheduler {
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
createAppAttemptId(1, 1), "default", "user1");
scheduler.handle(appAddedEvent);
- assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size());
- assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size());
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ .getAppSchedulables().size());
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("default")
+ .getAppSchedulables().size());
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
scheduler.reinitialize(conf, resourceManager.getRMContext());
AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
createAppAttemptId(2, 1), "default", "user2");
scheduler.handle(appAddedEvent2);
- assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size());
- assertEquals(1, scheduler.getQueueManager().getQueue("default").getApplications().size());
- assertEquals(0, scheduler.getQueueManager().getQueue("user2").getApplications().size());
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ .getAppSchedulables().size());
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("default")
+ .getAppSchedulables().size());
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
+ .getAppSchedulables().size());
}
@Test
@@ -338,18 +397,17 @@ public class TestFairScheduler {
scheduler.update();
- Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+ Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
- for (FSQueue p : queues) {
- if (p.getName().equals("queueA")) {
- assertEquals(1024, p.getQueueSchedulable().getFairShare().getMemory());
+ for (FSLeafQueue p : queues) {
+ if (p.getName().equals("root.queueA")) {
+ assertEquals(1024, p.getFairShare().getMemory());
}
- else if (p.getName().equals("queueB")) {
- assertEquals(2048, p.getQueueSchedulable().getFairShare().getMemory());
+ else if (p.getName().equals("root.queueB")) {
+ assertEquals(2048, p.getFairShare().getMemory());
}
}
-
}
/**
@@ -358,11 +416,11 @@ public class TestFairScheduler {
@Test
public void testQueueDemandCalculation() throws Exception {
ApplicationAttemptId id11 = createAppAttemptId(1, 1);
- scheduler.addApplication(id11, "queue1", "user1");
+ scheduler.addApplication(id11, "root.queue1", "user1");
ApplicationAttemptId id21 = createAppAttemptId(2, 1);
- scheduler.addApplication(id21, "queue2", "user1");
+ scheduler.addApplication(id21, "root.queue2", "user1");
ApplicationAttemptId id22 = createAppAttemptId(2, 2);
- scheduler.addApplication(id22, "queue2", "user1");
+ scheduler.addApplication(id22, "root.queue2", "user1");
int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB;
@@ -388,10 +446,10 @@ public class TestFairScheduler {
scheduler.update();
- assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1")
- .getQueueSchedulable().getDemand().getMemory());
+ assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1")
+ .getDemand().getMemory());
assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler
- .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand()
+ .getQueueManager().getQueue("root.queue2").getDemand()
.getMemory());
}
@@ -402,10 +460,11 @@ public class TestFairScheduler {
scheduler.handle(appAddedEvent1);
// Scheduler should have two queues (the default and the one created for user1)
- assertEquals(2, scheduler.getQueueManager().getQueues().size());
+ assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
// That queue should have one app
- assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size());
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ .getAppSchedulables().size());
AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);
@@ -414,7 +473,8 @@ public class TestFairScheduler {
scheduler.handle(appRemovedEvent1);
// Queue should have no apps
- assertEquals(0, scheduler.getQueueManager().getQueue("user1").getApplications().size());
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1")
+ .getAppSchedulables().size());
}
@Test
@@ -466,61 +526,99 @@ public class TestFairScheduler {
QueueManager queueManager = scheduler.getQueueManager();
queueManager.initialize();
- assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue
+ assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue
assertEquals(Resources.createResource(0),
- queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+ queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+ queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(1024),
- queueManager.getMinResources("queueA"));
+ queueManager.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048),
- queueManager.getMinResources("queueB"));
+ queueManager.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueC"));
+ queueManager.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueD"));
+ queueManager.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueE"));
+ queueManager.getMinResources("root.queueE"));
- assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME));
- assertEquals(15, queueManager.getQueueMaxApps("queueA"));
- assertEquals(15, queueManager.getQueueMaxApps("queueB"));
- assertEquals(15, queueManager.getQueueMaxApps("queueC"));
- assertEquals(3, queueManager.getQueueMaxApps("queueD"));
- assertEquals(15, queueManager.getQueueMaxApps("queueE"));
+ assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueA"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueB"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueC"));
+ assertEquals(3, queueManager.getQueueMaxApps("root.queueD"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueE"));
assertEquals(10, queueManager.getUserMaxApps("user1"));
assertEquals(5, queueManager.getUserMaxApps("user2"));
// Unspecified queues should get default ACL
- Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
+ Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB");
+ Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
// Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC");
+ Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout(
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
- assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+ assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
}
@Test
+ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
+ AllocationConfigurationException, ParserConfigurationException {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<minResources>2048</minResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<minResources>2048</minResources>");
+ out.println("<queue name=\"queueC\">");
+ out.println("<minResources>2048</minResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueD\">");
+ out.println("<minResources>2048</minResources>");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
+ Assert.assertEquals(4, leafQueues.size());
+ Assert.assertNotNull(queueManager.getLeafQueue("queueA"));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC"));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD"));
+ Assert.assertNotNull(queueManager.getLeafQueue("default"));
+ // Make sure querying for queues didn't create any new ones:
+ Assert.assertEquals(4, leafQueues.size());
+ }
+
+ @Test
public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
@@ -569,29 +667,29 @@ public class TestFairScheduler {
QueueManager queueManager = scheduler.getQueueManager();
queueManager.initialize();
- assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue
+ assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue
assertEquals(Resources.createResource(0),
- queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+ queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME));
+ queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(1024),
- queueManager.getMinResources("queueA"));
+ queueManager.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048),
- queueManager.getMinResources("queueB"));
+ queueManager.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueC"));
+ queueManager.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueD"));
+ queueManager.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
- queueManager.getMinResources("queueE"));
+ queueManager.getMinResources("root.queueE"));
- assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME));
- assertEquals(15, queueManager.getQueueMaxApps("queueA"));
- assertEquals(15, queueManager.getQueueMaxApps("queueB"));
- assertEquals(15, queueManager.getQueueMaxApps("queueC"));
- assertEquals(3, queueManager.getQueueMaxApps("queueD"));
- assertEquals(15, queueManager.getQueueMaxApps("queueE"));
+ assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueA"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueB"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueC"));
+ assertEquals(3, queueManager.getQueueMaxApps("root.queueD"));
+ assertEquals(15, queueManager.getQueueMaxApps("root.queueE"));
assertEquals(10, queueManager.getUserMaxApps("user1"));
assertEquals(5, queueManager.getUserMaxApps("user2"));
@@ -603,23 +701,23 @@ public class TestFairScheduler {
assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB");
+ Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
// Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC");
+ Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout(
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD"));
- assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA"));
- assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD"));
+ assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
+ assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
}
@@ -659,25 +757,25 @@ public class TestFairScheduler {
// Queue B arrives and wants 1 * 1024
createSchedulingRequest(1 * 1024, "queueB", "user1");
scheduler.update();
- Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+ Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
// Queue A should be above min share, B below.
- for (FSQueue p : queues) {
- if (p.getName().equals("queueA")) {
- assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+ for (FSLeafQueue p : queues) {
+ if (p.getName().equals("root.queueA")) {
+ assertEquals(false, scheduler.isStarvedForMinShare(p));
}
- else if (p.getName().equals("queueB")) {
- assertEquals(true, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+ else if (p.getName().equals("root.queueB")) {
+ assertEquals(true, scheduler.isStarvedForMinShare(p));
}
}
// Node checks in again, should allocate for B
scheduler.handle(nodeEvent2);
// Now B should have min share ( = demand here)
- for (FSQueue p : queues) {
- if (p.getName().equals("queueB")) {
- assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable()));
+ for (FSLeafQueue p : queues) {
+ if (p.getName().equals("root.queueB")) {
+ assertEquals(false, scheduler.isStarvedForMinShare(p));
}
}
}
@@ -718,16 +816,16 @@ public class TestFairScheduler {
// Queue B arrives and wants 1 * 1024
createSchedulingRequest(1 * 1024, "queueB", "user1");
scheduler.update();
- Collection<FSQueue> queues = scheduler.getQueueManager().getQueues();
+ Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
// Queue A should be above fair share, B below.
- for (FSQueue p : queues) {
- if (p.getName().equals("queueA")) {
- assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+ for (FSLeafQueue p : queues) {
+ if (p.getName().equals("root.queueA")) {
+ assertEquals(false, scheduler.isStarvedForFairShare(p));
}
- else if (p.getName().equals("queueB")) {
- assertEquals(true, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+ else if (p.getName().equals("root.queueB")) {
+ assertEquals(true, scheduler.isStarvedForFairShare(p));
}
}
@@ -735,9 +833,9 @@ public class TestFairScheduler {
scheduler.handle(nodeEvent2);
// B should not be starved for fair share, since entire demand is
// satisfied.
- for (FSQueue p : queues) {
- if (p.getName().equals("queueB")) {
- assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable()));
+ for (FSLeafQueue p : queues) {
+ if (p.getName().equals("root.queueB")) {
+ assertEquals(false, scheduler.isStarvedForFairShare(p));
}
}
}
@@ -845,7 +943,7 @@ public class TestFairScheduler {
// We should be able to claw back one container from A and B each.
// Make sure it is lowest priority container.
- scheduler.preemptResources(scheduler.getQueueSchedulables(),
+ scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
assertEquals(1, scheduler.applications.get(app2).getLiveContainers().size());
@@ -856,7 +954,7 @@ public class TestFairScheduler {
// We should be able to claw back another container from A and B each.
// Make sure it is lowest priority container.
- scheduler.preemptResources(scheduler.getQueueSchedulables(),
+ scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size());
@@ -866,7 +964,7 @@ public class TestFairScheduler {
assertEquals(0, scheduler.applications.get(app6).getLiveContainers().size());
// Now A and B are below fair share, so preemption shouldn't do anything
- scheduler.preemptResources(scheduler.getQueueSchedulables(),
+ scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size());
assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size());
@@ -977,10 +1075,10 @@ public class TestFairScheduler {
scheduler.update();
- FSQueueSchedulable schedC =
- scheduler.getQueueManager().getQueue("queueC").getQueueSchedulable();
- FSQueueSchedulable schedD =
- scheduler.getQueueManager().getQueue("queueD").getQueueSchedulable();
+ FSLeafQueue schedC =
+ scheduler.getQueueManager().getLeafQueue("queueC");
+ FSLeafQueue schedD =
+ scheduler.getQueueManager().getLeafQueue("queueD");
assertTrue(Resources.equals(
Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1415592&r1=1415591&r2=1415592&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Nov 30 12:03:25 2012
@@ -53,6 +53,22 @@ Hadoop MapReduce Next Generation - Fair
capacity between the running apps. queues can also be given weights to share
the cluster non-proportionally in the config file.
+ The fair scheduler supports hierarchical queues. All queues descend from a
+ queue named "root". Available resources are distributed among the children
+ of the root queue in the typical fair scheduling fashion. Then, the children
+ distribute the resources assigned to them to their children in the same
+ fashion. Applications may only be scheduled on leaf queues. Queues can be
+ specified as children of other queues by placing them as sub-elements of
+ their parents in the fair scheduler configuration file.
+
+ A queue's name starts with the names of its parents, with periods as
+ separators. So a queue named "queue1" under the root named, would be
+ referred to as "root.queue1", and a queue named "queue2" under a queue
+ named "parent1" would be referred to as "root.parent1.queue2". When
+ referring to queues, the root part of the name is optional, so queue1 could
+ be referred to as just "queue1", and a queue2 could be referred to as just
+ "parent1.queue2".
+
In addition to providing fair sharing, the Fair Scheduler allows assigning
guaranteed minimum shares to queues, which is useful for ensuring that
certain users, groups or production applications always get sufficient
@@ -163,11 +179,14 @@ Allocation file format
<?xml version="1.0"?>
<allocations>
<queue name="sample_queue">
- <minResources>100000</minResources>
- <maxResources>900000</maxResources>
+ <minResources>10000</minResources>
+ <maxResources>90000</maxResources>
<maxRunningApps>50</maxRunningApps>
<weight>2.0</weight>
<schedulingMode>fair</schedulingMode>
+ <queue name="sample_sub_queue">
+ <minResources>5000</minResources>
+ </queue>
</queue>
<user name="sample_user">
<maxRunningApps>30</maxRunningApps>