You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/03/27 19:38:29 UTC
svn commit: r1461773 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server/hado...
Author: vinodkv
Date: Wed Mar 27 18:38:28 2013
New Revision: 1461773
URL: http://svn.apache.org/r1461773
Log:
YARN-209. Fix CapacityScheduler to trigger application-activation when the cluster capacity changes. Contributed by Zhijie Shen.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1461773&r1=1461772&r2=1461773&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Mar 27 18:38:28 2013
@@ -149,6 +149,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-496. Fair scheduler configs are refreshed inconsistently in
reinitialize. (Sandy Ryza via tomwhite)
+ YARN-209. Fix CapacityScheduler to trigger application-activation when
+ the cluster capacity changes. (Zhijie Shen via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1461773&r1=1461772&r2=1461773&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Mar 27 18:38:28 2013
@@ -1481,7 +1481,11 @@ public class LeafQueue implements CSQueu
CSQueueUtils.updateQueueStatistics(
resourceCalculator, this, getParent(), clusterResource,
minimumAllocation);
-
+
+ // queue metrics are updated, more resource may be available
+ // activate the pending applications if possible
+ activateApplications();
+
// Update application properties
for (FiCaSchedulerApp application : activeApplications) {
synchronized (application) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java?rev=1461773&r1=1461772&r2=1461773&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java Wed Mar 27 18:38:28 2013
@@ -26,10 +26,12 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -135,6 +137,51 @@ public class TestRM {
rm.stop();
}
+ @Test (timeout = 30000)
+ public void testActivatingApplicationAfterAddingNM() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+
+ MockRM rm1 = new MockRM(conf);
+
+ // start like normal because state is empty
+ rm1.start();
+
+ // app that gets launched
+ RMApp app1 = rm1.submitApp(200);
+
+ // app that does not get launched
+ RMApp app2 = rm1.submitApp(200);
+
+ // app1 and app2 should be scheduled, but because no resource is available,
+ // they are not activated.
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
+ rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
+ RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
+ ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId();
+ rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);
+
+ MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
+ MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ nm2.registerNode();
+
+ //kick the scheduling
+ nm1.nodeHeartbeat(true);
+
+ // app1 should be allocated now
+ rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
+ rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);
+
+ nm2.nodeHeartbeat(true);
+
+ // app2 should be allocated now
+ rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
+ rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED);
+
+ rm1.stop();
+ }
+
public static void main(String[] args) throws Exception {
TestRM t = new TestRM();
t.testGetNewAppId();
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1461773&r1=1461772&r2=1461773&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Wed Mar 27 18:38:28 2013
@@ -1625,6 +1625,49 @@ public class TestLeafQueue {
assertEquals(0, e.pendingApplications.size());
}
+ @Test (timeout = 30000)
+ public void testActivateApplicationByUpdatingClusterResource()
+ throws Exception {
+
+ // Manipulate queue 'e'
+ LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E));
+
+ // Users
+ final String user_e = "user_e";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 =
+ TestUtils.getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app_0 =
+ new FiCaSchedulerApp(appAttemptId_0, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_0, user_e, E);
+
+ final ApplicationAttemptId appAttemptId_1 =
+ TestUtils.getMockApplicationAttemptId(1, 0);
+ FiCaSchedulerApp app_1 =
+ new FiCaSchedulerApp(appAttemptId_1, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_1, user_e, E); // same user
+
+ final ApplicationAttemptId appAttemptId_2 =
+ TestUtils.getMockApplicationAttemptId(2, 0);
+ FiCaSchedulerApp app_2 =
+ new FiCaSchedulerApp(appAttemptId_2, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_2, user_e, E); // same user
+
+ // before updating cluster resource
+ assertEquals(2, e.activeApplications.size());
+ assertEquals(1, e.pendingApplications.size());
+
+ e.updateClusterResource(Resources.createResource(200 * 16 * GB, 100 * 32));
+
+ // after updating cluster resource
+ assertEquals(3, e.activeApplications.size());
+ assertEquals(0, e.pendingApplications.size());
+ }
+
public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) {
for (QueueUserACLInfo aclInfo : aclInfos) {
if (aclInfo.getUserAcls().contains(acl)) {