You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/08/07 22:01:14 UTC
svn commit: r1616581 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/ap...
Author: jianhe
Date: Thu Aug 7 20:01:14 2014
New Revision: 1616581
URL: http://svn.apache.org/r1616581
Log:
Merge r1616580 from trunk. YARN-2008. Fixed CapacityScheduler to calculate headroom based on max available capacity instead of configured max capacity. Contributed by Craig Welch
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
- copied unchanged from r1616580, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Aug 7 20:01:14 2014
@@ -127,6 +127,9 @@ Release 2.6.0 - UNRELEASED
YARN-2388. Fixed TestTimelineWebServices failure due to HADOOP-10791. (zjshen)
+ YARN-2008. Fixed CapacityScheduler to calculate headroom based on max available
+ capacity instead of configured max capacity. (Craig Welch via jianhe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Thu Aug 7 20:01:14 2014
@@ -42,6 +42,13 @@ public class DefaultResourceCalculator e
Resource numerator, Resource denominator) {
return ratio(numerator, denominator);
}
+
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Thu Aug 7 20:01:14 2014
@@ -109,6 +109,14 @@ public class DominantResourceCalculator
getResourceAsValue(clusterResource, numerator, true) /
getResourceAsValue(clusterResource, denominator, true);
}
+
+ @Override
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java Thu Aug 7 20:01:14 2014
@@ -150,6 +150,15 @@ public abstract class ResourceCalculator
Resource clusterResource, Resource numerator, Resource denominator);
/**
+ * Determine if a resource is not suitable for use as a divisor
+ * (will result in divide by 0, etc)
+ *
+ * @param r resource
+ * @return true if divisor is invalid (should not be used), false else
+ */
+ public abstract boolean isInvalidDivisor(Resource r);
+
+ /**
* Ratio of resource <code>a</code> to resource <code>b</code>.
*
* @param a resource
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java Thu Aug 7 20:01:14 2014
@@ -184,6 +184,11 @@ public class Resources {
return calculator.roundDown(lhs, factor);
}
+ public static boolean isInvalidDivisor(
+ ResourceCalculator resourceCalculator, Resource divisor) {
+ return resourceCalculator.isInvalidDivisor(divisor);
+ }
+
public static float ratio(
ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
return resourceCalculator.ratio(lhs, rhs);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Thu Aug 7 20:01:14 2014
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -24,6 +27,8 @@ import org.apache.hadoop.yarn.util.resou
class CSQueueUtils {
+ private static final Log LOG = LogFactory.getLog(CSQueueUtils.class);
+
final static float EPSILON = 0.0001f;
public static void checkMaxCapacity(String queueName,
@@ -113,4 +118,52 @@ class CSQueueUtils {
)
);
}
+
+ public static float getAbsoluteMaxAvailCapacity(
+ ResourceCalculator resourceCalculator, Resource clusterResource, CSQueue queue) {
+ CSQueue parent = queue.getParent();
+ if (parent == null) {
+ return queue.getAbsoluteMaximumCapacity();
+ }
+
+ //Get my parent's max avail, needed to determine my own
+ float parentMaxAvail = getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, parent);
+ //...and as a resource
+ Resource parentResource = Resources.multiply(clusterResource, parentMaxAvail);
+
+ //check for no resources parent before dividing, if so, max avail is none
+ if (Resources.isInvalidDivisor(resourceCalculator, parentResource)) {
+ return 0.0f;
+ }
+ //sibling used is parent used - my used...
+ float siblingUsedCapacity = Resources.ratio(
+ resourceCalculator,
+ Resources.subtract(parent.getUsedResources(), queue.getUsedResources()),
+ parentResource);
+ //my max avail is the lesser of my max capacity and what is unused from my parent
+ //by my siblings (if they are beyond their base capacity)
+ float maxAvail = Math.min(
+ queue.getMaximumCapacity(),
+ 1.0f - siblingUsedCapacity);
+ //and, mutiply by parent to get absolute (cluster relative) value
+ float absoluteMaxAvail = maxAvail * parentMaxAvail;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("qpath " + queue.getQueuePath());
+ LOG.debug("parentMaxAvail " + parentMaxAvail);
+ LOG.debug("siblingUsedCapacity " + siblingUsedCapacity);
+ LOG.debug("getAbsoluteMaximumCapacity " + queue.getAbsoluteMaximumCapacity());
+ LOG.debug("maxAvail " + maxAvail);
+ LOG.debug("absoluteMaxAvail " + absoluteMaxAvail);
+ }
+
+ if (absoluteMaxAvail < 0.0f) {
+ absoluteMaxAvail = 0.0f;
+ } else if (absoluteMaxAvail > 1.0f) {
+ absoluteMaxAvail = 1.0f;
+ }
+
+ return absoluteMaxAvail;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1616581&r1=1616580&r2=1616581&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Aug 7 20:01:14 2014
@@ -976,13 +976,18 @@ public class LeafQueue implements CSQueu
Resource userLimit = // User limit
computeUserLimit(application, clusterResource, required);
-
+
+ //Max avail capacity needs to take into account usage by ancestor-siblings
+ //which are greater than their base capacity, so we are interested in "max avail"
+ //capacity
+ float absoluteMaxAvailCapacity = CSQueueUtils.getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, this);
Resource queueMaxCap = // Queue Max-Capacity
Resources.multiplyAndNormalizeDown(
resourceCalculator,
clusterResource,
- absoluteMaxCapacity,
+ absoluteMaxAvailCapacity,
minimumAllocation);
Resource userConsumed = getUser(user).getConsumedResources();