You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/12/09 02:44:04 UTC
svn commit: r1212213 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/had...
Author: mahadev
Date: Fri Dec 9 01:44:04 2011
New Revision: 1212213
URL: http://svn.apache.org/viewvc?rev=1212213&view=rev
Log:
MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value for queues (Anupam Seth via mahadev) - Merging r1212212 from trunk
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Dec 9 01:44:04 2011
@@ -203,6 +203,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3147. Handle leaf queues with the same name properly.
(Ravi Prakash via mahadev)
+ MAPREDUCE-3327. RM web ui scheduler link doesn't show correct max value
+ for queues (Anupam Seth via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Fri Dec 9 01:44:04 2011
@@ -21,7 +21,7 @@ class CSQueueUtils {
public static void checkMaxCapacity(String queueName,
float capacity, float maximumCapacity) {
- if (maximumCapacity != CapacitySchedulerConfiguration.UNDEFINED &&
+ if (Math.round(100 * maximumCapacity) != CapacitySchedulerConfiguration.UNDEFINED &&
maximumCapacity < capacity) {
throw new IllegalArgumentException(
"Illegal call to setMaxCapacity. " +
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Fri Dec 9 01:44:04 2011
@@ -162,6 +162,8 @@ public class CapacitySchedulerConfigurat
public void setMaximumCapacity(String queue, int maxCapacity) {
setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
+ LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
+ ", maxCapacity=" + maxCapacity);
}
public int getUserLimit(String queue) {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Dec 9 01:44:04 2011
@@ -146,10 +146,10 @@ public class LeafQueue implements CSQueu
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
- float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
+ float maximumCapacity = (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
float absoluteMaxCapacity =
- (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
- Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100;
+ (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
+ Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity);
int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
float userLimitFactor =
@@ -402,7 +402,7 @@ public class LeafQueue implements CSQueu
this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity =
- (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
+ (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
Float.MAX_VALUE :
(parent.getAbsoluteCapacity() * maximumCapacity);
}
@@ -829,13 +829,13 @@ public class LeafQueue implements CSQueu
float potentialNewCapacity =
(float)(usedResources.getMemory() + required.getMemory()) /
clusterResource.getMemory();
+ LOG.info(getQueueName() +
+ " usedResources: " + usedResources.getMemory() +
+ " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
+ " required " + required.getMemory() +
+ " potentialNewCapacity: " + potentialNewCapacity + " ( " +
+ " max-capacity: " + absoluteMaxCapacity + ")");
if (potentialNewCapacity > absoluteMaxCapacity) {
- LOG.info(getQueueName() +
- " usedResources: " + usedResources.getMemory() +
- " currentCapacity " + ((float)usedResources.getMemory())/clusterResource.getMemory() +
- " required " + required.getMemory() +
- " potentialNewCapacity: " + potentialNewCapacity + " ( " +
- " > max-capacity (" + absoluteMaxCapacity + ")");
return false;
}
return true;
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Fri Dec 9 01:44:04 2011
@@ -123,10 +123,10 @@ public class ParentQueue implements CSQu
float absoluteCapacity = parentAbsoluteCapacity * capacity;
float maximumCapacity =
- cs.getConfiguration().getMaximumCapacity(getQueuePath());
+ (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
float absoluteMaxCapacity =
- (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
- 1000000000f : (parentAbsoluteCapacity * maximumCapacity) / 100;
+ (Math.round(maximumCapacity * 100) == CapacitySchedulerConfiguration.UNDEFINED) ?
+ Float.MAX_VALUE : (parentAbsoluteCapacity * maximumCapacity);
QueueState state = cs.getConfiguration().getState(getQueuePath());
@@ -233,12 +233,12 @@ public class ParentQueue implements CSQu
@Override
public float getAbsoluteMaximumCapacity() {
- return 0;
+ return absoluteMaxCapacity;
}
@Override
public float getMaximumCapacity() {
- return 0;
+ return maximumCapacity;
}
@Override
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1212213&r1=1212212&r2=1212213&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Fri Dec 9 01:44:04 2011
@@ -107,12 +107,15 @@ public class TestLeafQueue {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
final String Q_A = CapacityScheduler.ROOT + "." + A;
conf.setCapacity(Q_A, 10);
+ conf.setMaximumCapacity(Q_A, 20);
final String Q_B = CapacityScheduler.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
+ conf.setMaximumCapacity(Q_B, 99);
LOG.info("Setup top-level queues a and b");
}
@@ -158,6 +161,23 @@ public class TestLeafQueue {
return queue;
}
+ @Test
+ public void testInitializeQueue() throws Exception {
+ final float epsilon = 1e-5f;
+ //can add more sturdy test with 3-layer queues
+ //once MAPREDUCE:3410 is resolved
+ LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ assertEquals(0.1, a.getCapacity(), epsilon);
+ assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.2, a.getMaximumCapacity(), epsilon);
+ assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+
+ LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+ assertEquals(0.9, b.getCapacity(), epsilon);
+ assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.99, b.getMaximumCapacity(), epsilon);
+ assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+ }
@Test
public void testSingleQueueOneUserMetrics() throws Exception {
@@ -209,6 +229,8 @@ public class TestLeafQueue {
// Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ //unset maxCapacity
+ a.setMaxCapacity(-0.01f);
// Users
final String user_0 = "user_0";
@@ -329,6 +351,8 @@ public class TestLeafQueue {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ //unset maxCapacity
+ a.setMaxCapacity(-0.01f);
// Users
final String user_0 = "user_0";
@@ -442,7 +466,7 @@ public class TestLeafQueue {
// Revert max-capacity and user-limit-factor
// Now, allocations should goto app_3 since it's under user-limit
- a.setMaxCapacity(-1);
+ a.setMaxCapacity(-0.01f);
a.setUserLimitFactor(1);
a.assignContainers(clusterResource, node_0);
assertEquals(7*GB, a.getUsedResources().getMemory());
@@ -498,6 +522,8 @@ public class TestLeafQueue {
// Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ //unset maxCapacity
+ a.setMaxCapacity(-0.01f);
// Users
final String user_0 = "user_0";
@@ -594,6 +620,8 @@ public class TestLeafQueue {
// Manipulate queue 'a'
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ //unset maxCapacity
+ a.setMaxCapacity(-0.01f);
a.setUserLimitFactor(10);
// Users