You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/19 04:42:18 UTC
svn commit: r1469669 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/proto/
hadoop-yarn/hadoop-yarn-common/src/main/...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Apr 19 02:42:16 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -72,7 +73,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
import org.junit.Before;
@@ -283,7 +284,7 @@ public class TestFairScheduler {
assertEquals(capacity / 4, queue2.getFairShare().getMemory());
assertEquals(capacity / 4, queue3.getFairShare().getMemory());
}
-
+
@Test
public void testHierarchicalQueuesSimilarParents() {
QueueManager queueManager = scheduler.getQueueManager();
@@ -1358,7 +1359,7 @@ public class TestFairScheduler {
FSSchedulerApp app2 = scheduler.applications.get(attId2);
FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1");
- queue1.setSchedulingMode(new FifoSchedulingMode());
+ queue1.setPolicy(new FifoPolicy());
scheduler.update();
@@ -1380,7 +1381,80 @@ public class TestFairScheduler {
assertEquals(2, app1.getLiveContainers().size());
assertEquals(1, app2.getLiveContainers().size());
}
-
+
+ /**
+ * Test to verify the behavior of
+ * {@link FSQueue#assignContainer(FSSchedulerNode)})
+ *
+ * Create two queues under root (fifoQueue and fairParent), and two queues
+ * under fairParent (fairChild1 and fairChild2). Submit two apps to the
+ * fifoQueue and one each to the fairChild* queues, all apps requiring 4
+ * containers each of the total 16 container capacity
+ *
+ * Assert the number of containers for each app after 4, 8, 12 and 16 updates.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 5000)
+ public void testAssignContainer() throws Exception {
+ final String user = "user1";
+ final String fifoQueue = "fifo";
+ final String fairParent = "fairParent";
+ final String fairChild1 = fairParent + ".fairChild1";
+ final String fairChild2 = fairParent + ".fairChild2";
+
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8192));
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(8192));
+
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+
+ scheduler.handle(nodeEvent1);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest(1024, fifoQueue, user, 4);
+ ApplicationAttemptId attId2 =
+ createSchedulingRequest(1024, fairChild1, user, 4);
+ ApplicationAttemptId attId3 =
+ createSchedulingRequest(1024, fairChild2, user, 4);
+ ApplicationAttemptId attId4 =
+ createSchedulingRequest(1024, fifoQueue, user, 4);
+
+ FSSchedulerApp app1 = scheduler.applications.get(attId1);
+ FSSchedulerApp app2 = scheduler.applications.get(attId2);
+ FSSchedulerApp app3 = scheduler.applications.get(attId3);
+ FSSchedulerApp app4 = scheduler.applications.get(attId4);
+
+ scheduler.getQueueManager().getLeafQueue(fifoQueue)
+ .setPolicy(SchedulingPolicy.parse("fifo"));
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
+
+ for (int i = 0; i < 8; i++) {
+ scheduler.handle(updateEvent1);
+ scheduler.handle(updateEvent2);
+ if ((i + 1) % 2 == 0) {
+ // 4 node updates: fifoQueue should have received 2, and fairChild*
+ // should have received one each
+ String ERR =
+ "Wrong number of assigned containers after " + (i + 1) + " updates";
+ if (i < 4) {
+ // app1 req still not met
+ assertEquals(ERR, (i + 1), app1.getLiveContainers().size());
+ assertEquals(ERR, 0, app4.getLiveContainers().size());
+ } else {
+ // app1 req has been met, app4 should be served now
+ assertEquals(ERR, 4, app1.getLiveContainers().size());
+ assertEquals(ERR, (i - 3), app4.getLiveContainers().size());
+ }
+ assertEquals(ERR, (i + 1) / 2, app2.getLiveContainers().size());
+ assertEquals(ERR, (i + 1) / 2, app3.getLiveContainers().size());
+ }
+ }
+ }
@SuppressWarnings("unchecked")
@Test
@@ -1411,6 +1485,7 @@ public class TestFairScheduler {
ContainerLaunchContext clc =
BuilderUtils.newContainerLaunchContext(user, null, null, null, null,
null, null);
+ submissionContext.setApplicationId(applicationId);
submissionContext.setAMContainerSpec(clc);
RMApp application =
new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user,
@@ -1419,13 +1494,24 @@ public class TestFairScheduler {
resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application);
application.handle(new RMAppEvent(applicationId, RMAppEventType.START));
+ final int MAX_TRIES=20;
+ int numTries = 0;
+ while (!application.getState().equals(RMAppState.SUBMITTED) &&
+ numTries < MAX_TRIES) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ex) {ex.printStackTrace();}
+ numTries++;
+ }
+ assertEquals("The application doesn't reach SUBMITTED.",
+ RMAppState.SUBMITTED, application.getState());
+
ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
attId.setAttemptId(this.ATTEMPT_ID++);
attId.setApplicationId(applicationId);
scheduler.addApplication(attId, queue, user);
-
- final int MAX_TRIES=20;
- int numTries = 0;
+
+ numTries = 0;
while (application.getFinishTime() == 0 && numTries < MAX_TRIES) {
try {
Thread.sleep(100);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Fri Apr 19 02:42:16 2013
@@ -47,6 +47,8 @@ MapReduce NextGen aka YARN aka MRv2
* {{{./CapacityScheduler.html}Capacity Scheduler}}
+ * {{{./FairScheduler.html}Fair Scheduler}}
+
* {{{./WebApplicationProxy.html}Web Application Proxy}}
* {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}}