You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/03/27 04:44:35 UTC
svn commit: r1461402 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server/hado...
Author: vinodkv
Date: Wed Mar 27 03:44:34 2013
New Revision: 1461402
URL: http://svn.apache.org/r1461402
Log:
YARN-474. Fix CapacityScheduler to trigger application-activation when am-resource-percent configuration is refreshed. Contributed by Zhijie Shen.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1461402&r1=1461401&r2=1461402&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Mar 27 03:44:34 2013
@@ -143,6 +143,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-498. Unmanaged AM launcher does not set various constants in env for
an AM, also does not handle failed AMs properly. (Hitesh Shah via bikas)
+ YARN-474. Fix CapacityScheduler to trigger application-activation when
+ am-resource-percent configuration is refreshed. (Zhijie Shen via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1461402&r1=1461401&r2=1461402&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Mar 27 03:44:34 2013
@@ -607,6 +607,10 @@ public class LeafQueue implements CSQueu
newlyParsedLeafQueue.getMaximumActiveApplications(),
newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
+
+ // queue metrics are updated, more resource may be available
+ // activate the pending applications if possible
+ activateApplications();
}
@Override
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1461402&r1=1461401&r2=1461402&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Wed Mar 27 03:44:34 2013
@@ -138,6 +138,7 @@ public class TestLeafQueue {
private static final String C = "c";
private static final String C1 = "c1";
private static final String D = "d";
+ private static final String E = "e";
private void setupQueueConfiguration(
CapacitySchedulerConfiguration conf,
final String newRoot) {
@@ -148,7 +149,7 @@ public class TestLeafQueue {
conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot;
- conf.setQueues(Q_newRoot, new String[] {A, B, C, D});
+ conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E});
conf.setCapacity(Q_newRoot, 100);
conf.setMaximumCapacity(Q_newRoot, 100);
conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " ");
@@ -174,10 +175,14 @@ public class TestLeafQueue {
conf.setCapacity(Q_C1, 100);
final String Q_D = Q_newRoot + "." + D;
- conf.setCapacity(Q_D, 10);
+ conf.setCapacity(Q_D, 9);
conf.setMaximumCapacity(Q_D, 11);
conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d");
+ final String Q_E = Q_newRoot + "." + E;
+ conf.setCapacity(Q_E, 1);
+ conf.setMaximumCapacity(Q_E, 1);
+ conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e");
}
static LeafQueue stubLeafQueue(LeafQueue queue) {
@@ -1567,6 +1572,59 @@ public class TestLeafQueue {
}
+ @Test (timeout = 30000)
+ public void testActivateApplicationAfterQueueRefresh() throws Exception {
+
+ // Manipulate queue 'e'
+ LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E));
+
+ // Users
+ final String user_e = "user_e";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 =
+ TestUtils.getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app_0 =
+ new FiCaSchedulerApp(appAttemptId_0, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_0, user_e, E);
+
+ final ApplicationAttemptId appAttemptId_1 =
+ TestUtils.getMockApplicationAttemptId(1, 0);
+ FiCaSchedulerApp app_1 =
+ new FiCaSchedulerApp(appAttemptId_1, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_1, user_e, E); // same user
+
+ final ApplicationAttemptId appAttemptId_2 =
+ TestUtils.getMockApplicationAttemptId(2, 0);
+ FiCaSchedulerApp app_2 =
+ new FiCaSchedulerApp(appAttemptId_2, user_e, e,
+ mock(ActiveUsersManager.class), rmContext);
+ e.submitApplication(app_2, user_e, E); // same user
+
+ // before reinitialization
+ assertEquals(2, e.activeApplications.size());
+ assertEquals(1, e.pendingApplications.size());
+
+ csConf.setDouble(CapacitySchedulerConfiguration
+ .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot =
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT,
+ newQueues, queues,
+ TestUtils.spyHook);
+ queues = newQueues;
+ root.reinitialize(newRoot, cs.getClusterResources());
+
+ // after reinitialization
+ assertEquals(3, e.activeApplications.size());
+ assertEquals(0, e.pendingApplications.size());
+ }
+
public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) {
for (QueueUserACLInfo aclInfo : aclInfos) {
if (aclInfo.getUserAcls().contains(acl)) {