You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2012/03/05 00:44:57 UTC
svn commit: r1296898 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server...
Author: acmurthy
Date: Sun Mar 4 23:44:57 2012
New Revision: 1296898
URL: http://svn.apache.org/viewvc?rev=1296898&view=rev
Log:
MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by using capacity and not max-capacity since we are already scaling it by userLimitFactor. Contributed by Eric Payne.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sun Mar 4 23:44:57 2012
@@ -253,6 +253,10 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured
hostname or IP. (tgraves via acmurthy)
+ MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by
+ using capacity and not max-capacity since we are already scaling it by
+ userLimitFactor. (Eric Payne via acmurthy)
+
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Sun Mar 4 23:44:57 2012
@@ -88,7 +88,8 @@ public class LeafQueue implements CSQueu
private int maxApplicationsPerUser;
private float maxAMResourcePercent;
- private int maxActiveApplications;
+ private int maxActiveApplications; // Based on absolute max capacity
+ private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
private Resource usedResources = Resources.createResource(0);
@@ -167,8 +168,12 @@ public class LeafQueue implements CSQueu
CSQueueUtils.computeMaxActiveApplications(
cs.getClusterResources(), this.minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
+ this.maxActiveAppsUsingAbsCap =
+ CSQueueUtils.computeMaxActiveApplications(
+ cs.getClusterResources(), this.minimumAllocation,
+ maxAMResourcePercent, absoluteCapacity);
int maxActiveApplicationsPerUser =
- CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit,
+ CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit,
userLimitFactor);
this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
@@ -271,6 +276,11 @@ public class LeafQueue implements CSQueu
"(int)ceil((clusterResourceMemory / minimumAllocation) *" +
"maxAMResourcePercent * absoluteMaxCapacity)," +
"1) ]" + "\n" +
+ "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
+ " [= max(" +
+ "(int)ceil((clusterResourceMemory / minimumAllocation) *" +
+ "maxAMResourcePercent * absoluteCapacity)," +
+ "1) ]" + "\n" +
"maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
" [= max(" +
"(int)(maxActiveApplications * (userLimit / 100.0f) * " +
@@ -1376,9 +1386,13 @@ public class LeafQueue implements CSQueu
CSQueueUtils.computeMaxActiveApplications(
clusterResource, minimumAllocation,
maxAMResourcePercent, absoluteMaxCapacity);
+ maxActiveAppsUsingAbsCap =
+ CSQueueUtils.computeMaxActiveApplications(
+ clusterResource, minimumAllocation,
+ maxAMResourcePercent, absoluteCapacity);
maxActiveApplicationsPerUser =
CSQueueUtils.computeMaxActiveApplicationsPerUser(
- maxActiveApplications, userLimit, userLimitFactor);
+ maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
// Update metrics
CSQueueUtils.updateQueueStatistics(
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Sun Mar 4 23:44:57 2012
@@ -158,9 +158,14 @@ public class TestApplicationLimits {
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
+ int expectedMaxActiveAppsUsingAbsCap =
+ Math.max(1,
+ (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
+ csConf.getMaximumApplicationMasterResourcePercent() *
+ queue.getAbsoluteCapacity()));
assertEquals(
(int)Math.ceil(
- expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) *
+ expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) *
queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(
@@ -178,8 +183,13 @@ public class TestApplicationLimits {
queue.getAbsoluteMaximumCapacity()));
assertEquals(expectedMaxActiveApps,
queue.getMaximumActiveApplications());
+ expectedMaxActiveAppsUsingAbsCap =
+ Math.max(1,
+ (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
+ csConf.getMaximumApplicationMasterResourcePercent() *
+ queue.getAbsoluteCapacity()));
assertEquals(
- (int)Math.ceil(expectedMaxActiveApps *
+ (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap *
(queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(