You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by jl...@apache.org on 2014/06/16 18:14:07 UTC
svn commit: r1602918 - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capac...
Author: jlowe
Date: Mon Jun 16 16:14:06 2014
New Revision: 1602918
URL: http://svn.apache.org/r1602918
Log:
YARN-853. maximum-am-resource-percent doesn't work after refreshQueues command. Contributed by Devaraj K
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1602918&r1=1602917&r2=1602918&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Mon Jun 16 16:14:06 2014
@@ -36,6 +36,9 @@ Release 0.23.11 - UNRELEASED
YARN-1932. Javascript injection on the job status page (Mit Desai via
jlowe)
+ YARN-853. maximum-am-resource-percent doesn't work after refreshQueues
+ command (Devaraj K via jlowe)
+
Release 0.23.10 - 2013-12-09
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1602918&r1=1602917&r2=1602918&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Mon Jun 16 16:14:06 2014
@@ -146,6 +146,7 @@
<Field name="state" />
<Field name="userLimit" />
<Field name="userLimitFactor" />
+ <Field name="maxAMResourcePerQueuePercent" />
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1602918&r1=1602917&r2=1602918&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Mon Jun 16 16:14:06 2014
@@ -166,9 +166,8 @@ public class LeafQueue implements CSQueu
maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
- this.maxAMResourcePerQueuePercent =
- cs.getConfiguration().
- getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
+ float maxAMResourcePerQueuePercent = cs.getConfiguration()
+ .getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
int maxActiveApplications =
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
@@ -195,9 +194,9 @@ public class LeafQueue implements CSQueu
capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity,
userLimit, userLimitFactor,
- maxApplications, maxApplicationsPerUser,
- maxActiveApplications, maxActiveApplicationsPerUser,
- state, acls, cs.getConfiguration().getNodeLocalityDelay());
+ maxApplications, maxAMResourcePerQueuePercent, maxApplicationsPerUser,
+ maxActiveApplications, maxActiveApplicationsPerUser, state, acls, cs
+ .getConfiguration().getNodeLocalityDelay());
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
@@ -214,10 +213,10 @@ public class LeafQueue implements CSQueu
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
int userLimit, float userLimitFactor,
- int maxApplications, int maxApplicationsPerUser,
- int maxActiveApplications, int maxActiveApplicationsPerUser,
- QueueState state, Map<QueueACL, AccessControlList> acls,
- int nodeLocalityDelay)
+ int maxApplications, float maxAMResourcePerQueuePercent,
+ int maxApplicationsPerUser, int maxActiveApplications,
+ int maxActiveApplicationsPerUser, QueueState state,
+ Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay)
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -234,6 +233,7 @@ public class LeafQueue implements CSQueu
this.userLimitFactor = userLimitFactor;
this.maxApplications = maxApplications;
+ this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
this.maxApplicationsPerUser = maxApplicationsPerUser;
this.maxActiveApplications = maxActiveApplications;
@@ -382,6 +382,14 @@ public class LeafQueue implements CSQueu
public float getMinimumAllocationFactor() {
return minimumAllocationFactor;
}
+
+ /**
+ * Used only by tests.
+ */
+ @Private
+ public float getMaxAMResourcePerQueuePercent() {
+ return maxAMResourcePerQueuePercent;
+ }
public int getMaxApplications() {
return maxApplications;
@@ -594,6 +602,7 @@ public class LeafQueue implements CSQueu
newlyParsedLeafQueue.absoluteMaxCapacity,
newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor,
newlyParsedLeafQueue.maxApplications,
+ newlyParsedLeafQueue.maxAMResourcePerQueuePercent,
newlyParsedLeafQueue.getMaxApplicationsPerUser(),
newlyParsedLeafQueue.getMaximumActiveApplications(),
newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1602918&r1=1602917&r2=1602918&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon Jun 16 16:14:06 2014
@@ -1614,6 +1614,51 @@ public class TestLeafQueue {
// after reinitialization
assertEquals(60, e.getNodeLocalityDelay());
}
+
+ @Test
+ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
+ throws Exception {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ Resource clusterResource = Resources
+ .createResource(100 * 16 * GB);
+ CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
+ csConf.setFloat(CapacitySchedulerConfiguration.
+ MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
+ ParentQueue root = new ParentQueue(csContext,
+ CapacitySchedulerConfiguration.ROOT, CapacityScheduler.queueComparator,
+ null, null);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
+ LeafQueue a = new LeafQueue(csContext, A, root,
+ CapacityScheduler.applicationComparator, null);
+ assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(160, a.getMaximumActiveApplications());
+
+ csConf.setFloat(CapacitySchedulerConfiguration.
+ MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
+ LeafQueue newA = new LeafQueue(csContext, A, root,
+ CapacityScheduler.applicationComparator, null);
+ a.reinitialize(newA, clusterResource);
+ assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(320, a.getMaximumActiveApplications());
+
+ Resource newClusterResource = Resources.createResource(100 * 20 * GB);
+ a.updateClusterResource(newClusterResource);
+ // 100 * 20 * 0.2 = 400
+ assertEquals(400, a.getMaximumActiveApplications());
+ }
+
+ private CapacitySchedulerContext mockCSContext(
+ CapacitySchedulerConfiguration csConf, Resource clusterResource) {
+ CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
+ when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getConf()).thenReturn(new YarnConfiguration());
+ when(csContext.getClusterResources()).thenReturn(clusterResource);
+ when(csContext.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(GB));
+ when(csContext.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(2 * GB));
+ return csContext;
+ }
@After