You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/11/23 02:42:52 UTC
svn commit: r1205260 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server...
Author: acmurthy
Date: Wed Nov 23 01:42:51 2011
New Revision: 1205260
URL: http://svn.apache.org/viewvc?rev=1205260&view=rev
Log:
MAPREDUCE-3329. Fixed CapacityScheduler to ensure maximum-capacity cannot be lesser than capacity for any queue.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1205260&r1=1205259&r2=1205260&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Nov 23 01:42:51 2011
@@ -170,6 +170,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3408. yarn-daemon.sh unconditionnaly sets yarn.root.logger
(Bruno Mahe via mahadev)
+ MAPREDUCE-3329. Fixed CapacityScheduler to ensure maximum-capacity cannot
+ be lesser than capacity for any queue. (acmurthy)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1205260&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Wed Nov 23 01:42:51 2011
@@ -0,0 +1,17 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+class CSQueueUtils {
+
+ public static void checkMaxCapacity(String queueName,
+ float capacity, float maximumCapacity) {
+ if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED &&
+ maximumCapacity < capacity) {
+ throw new IllegalArgumentException(
+ "Illegal call to setMaxCapacity. " +
+ "Queue '" + queueName + "' has " +
+ "capacity (" + capacity + ") greater than " +
+ "maximumCapacity (" + maximumCapacity + ")" );
+ }
+ }
+
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1205260&r1=1205259&r2=1205260&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Wed Nov 23 01:42:51 2011
@@ -160,6 +160,10 @@ public class CapacitySchedulerConfigurat
return maxCapacity;
}
+ public void setMaximumCapacity(String queue, int maxCapacity) {
+ setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
+ }
+
public int getUserLimit(String queue) {
int userLimit =
getInt(getQueuePrefix(queue) + USER_LIMIT, DEFAULT_USER_LIMIT);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1205260&r1=1205259&r2=1205260&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Nov 23 01:42:51 2011
@@ -211,16 +211,19 @@ public class LeafQueue implements CSQueu
private synchronized void setupQueueConfigs(
float capacity, float absoluteCapacity,
- float maxCapacity, float absoluteMaxCapacity,
+ float maximumCapacity, float absoluteMaxCapacity,
int userLimit, float userLimitFactor,
int maxApplications, int maxApplicationsPerUser,
int maxActiveApplications, int maxActiveApplicationsPerUser,
QueueState state, Map<QueueACL, AccessControlList> acls)
{
+ // Sanity check
+ CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+
this.capacity = capacity;
this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
- this.maximumCapacity = maxCapacity;
+ this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = absoluteMaxCapacity;
this.userLimit = userLimit;
@@ -236,9 +239,9 @@ public class LeafQueue implements CSQueu
this.acls = acls;
- this.queueInfo.setCapacity(capacity);
- this.queueInfo.setMaximumCapacity(maximumCapacity);
- this.queueInfo.setQueueState(state);
+ this.queueInfo.setCapacity(this.capacity);
+ this.queueInfo.setMaximumCapacity(this.maximumCapacity);
+ this.queueInfo.setQueueState(this.state);
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -250,7 +253,7 @@ public class LeafQueue implements CSQueu
" [= (float) configuredCapacity / 100 ]" + "\n" +
"asboluteCapacity = " + absoluteCapacity +
" [= parentAbsoluteCapacity * capacity ]" + "\n" +
- "maxCapacity = " + maxCapacity +
+ "maxCapacity = " + maximumCapacity +
" [= configuredMaxCapacity ]" + "\n" +
"absoluteMaxCapacity = " + absoluteMaxCapacity +
" [= Float.MAX_VALUE if maximumCapacity undefined, " +
@@ -394,6 +397,9 @@ public class LeafQueue implements CSQueu
* @param maximumCapacity new max capacity
*/
synchronized void setMaxCapacity(float maximumCapacity) {
+ // Sanity check
+ CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+
this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1205260&r1=1205259&r2=1205260&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Wed Nov 23 01:42:51 2011
@@ -153,6 +153,9 @@ public class ParentQueue implements CSQu
float maximumCapacity, float absoluteMaxCapacity,
QueueState state, Map<QueueACL, AccessControlList> acls
) {
+ // Sanity check
+ CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+
this.capacity = capacity;
this.absoluteCapacity = absoluteCapacity;
this.maximumCapacity = maximumCapacity;
@@ -162,9 +165,9 @@ public class ParentQueue implements CSQu
this.acls = acls;
- this.queueInfo.setCapacity(capacity);
- this.queueInfo.setMaximumCapacity(maximumCapacity);
- this.queueInfo.setQueueState(state);
+ this.queueInfo.setCapacity(this.capacity);
+ this.queueInfo.setMaximumCapacity(this.maximumCapacity);
+ this.queueInfo.setQueueState(this.state);
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -484,6 +487,9 @@ public class ParentQueue implements CSQu
* @param maximumCapacity new max capacity
*/
synchronized void setMaxCapacity(float maximumCapacity) {
+ // Sanity check
+ CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+
this.maximumCapacity = maximumCapacity;
float parentAbsoluteCapacity =
(rootQueue) ? 100.0f : parent.getAbsoluteCapacity();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1205260&r1=1205259&r2=1205260&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Wed Nov 23 01:42:51 2011
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -35,7 +37,6 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.reinitialize(conf, null, null);
- //capacityScheduler.g
}
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
@@ -104,4 +105,48 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.reinitialize(conf, null, null);
}
+
+ public void testMaxCapacity() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
+ conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
+ conf.setCapacity(CapacityScheduler.ROOT, 100);
+
+ final String A = CapacityScheduler.ROOT + ".a";
+ conf.setCapacity(A, 50);
+ conf.setMaximumCapacity(A, 60);
+
+ final String B = CapacityScheduler.ROOT + ".b";
+ conf.setCapacity(B, 50);
+ conf.setMaximumCapacity(B, 45); // Should throw an exception
+
+
+ boolean fail = false;
+ CapacityScheduler capacityScheduler;
+ try {
+ capacityScheduler = new CapacityScheduler();
+ capacityScheduler.reinitialize(conf, null, null);
+ } catch (IllegalArgumentException iae) {
+ fail = true;
+ }
+ Assert.assertTrue("Didn't throw IllegalArgumentException for wrong maxCap",
+ fail);
+
+ conf.setMaximumCapacity(B, 60);
+
+ // Now this should work
+ capacityScheduler = new CapacityScheduler();
+ capacityScheduler.reinitialize(conf, null, null);
+
+ fail = false;
+ try {
+ LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A);
+ a.setMaxCapacity(45);
+ } catch (IllegalArgumentException iae) {
+ fail = true;
+ }
+ Assert.assertTrue("Didn't throw IllegalArgumentException for wrong " +
+ "setMaxCap", fail);
+ }
+
}