You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/12/10 21:54:54 UTC
[21/50] [abbrv] hadoop git commit: YARN-7254. UI and metrics changes
related to absolute resource configuration. (Sunil G via wangda)
YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d52627a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d52627a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d52627a7
Branch: refs/heads/YARN-6592
Commit: d52627a7cbddfd981db973e223aefffde1ebf82d
Parents: 5e798b1
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 16 11:04:05 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800
----------------------------------------------------------------------
.../yarn/api/records/QueueConfigurations.java | 76 +++++++++++
.../yarn/api/records/ResourceInformation.java | 6 +
.../src/main/proto/yarn_protos.proto | 4 +
.../impl/pb/QueueConfigurationsPBImpl.java | 136 ++++++++++++++++++-
.../resource/DominantResourceCalculator.java | 24 +++-
.../scheduler/QueueResourceQuotas.java | 38 ------
.../scheduler/capacity/AbstractCSQueue.java | 18 ++-
.../scheduler/capacity/CSQueue.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/capacity/ParentQueue.java | 56 ++++++--
.../scheduler/capacity/UsersManager.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 23 +++-
.../webapp/CapacitySchedulerPage.java | 49 +++++--
.../webapp/dao/CapacitySchedulerInfo.java | 3 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 6 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 10 +-
.../dao/PartitionQueueCapacitiesInfo.java | 34 ++++-
.../webapp/dao/QueueCapacitiesInfo.java | 13 +-
...CapacitySchedulerWithMultiResourceTypes.java | 25 ++--
.../scheduler/capacity/TestLeafQueue.java | 24 +++-
20 files changed, 443 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
index e25c8aa..30096bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
@@ -147,4 +147,80 @@ public abstract class QueueConfigurations {
@Private
@Unstable
public abstract void setMaxAMPercentage(float maxAMPercentage);
+
+ /**
+ * Get the effective minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMinCapacity();
+
+ /**
+ * Set the effective minimum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * minimum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMinCapacity(Resource capacity);
+
+ /**
+ * Get the effective maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMaxCapacity();
+
+ /**
+ * Set the effective maximum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * maximum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMaxCapacity(Resource capacity);
+
+ /**
+ * Get the configured minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMinCapacity();
+
+ /**
+ * Set the configured minimum capacity of queue (from absolute resource).
+ *
+ * @param configuredMinResource
+ * minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMinCapacity(Resource configuredMinResource);
+
+ /**
+ * Get the configured maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMaxCapacity();
+
+ /**
+ * Set the configured maximum capacity of queue (from absolute resource).
+ *
+ * @param configuredMaxResource
+ * maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index a8198d8..904ff4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -228,6 +228,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
Long.MAX_VALUE);
}
+ public static ResourceInformation newInstance(String name, String units,
+ long minRes, long maxRes) {
+ return ResourceInformation.newInstance(name, units, 0L,
+ ResourceTypes.COUNTABLE, minRes, maxRes);
+ }
+
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fdbe2d4..3a9662b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -551,6 +551,10 @@ message QueueConfigurationsProto {
optional float maxCapacity = 3;
optional float absoluteMaxCapacity = 4;
optional float maxAMPercentage = 5;
+ optional ResourceProto effectiveMinCapacity = 6;
+ optional ResourceProto effectiveMaxCapacity = 7;
+ optional ResourceProto configuredMinCapacity = 8;
+ optional ResourceProto configuredMaxCapacity = 9;
}
message QueueConfigurationsMapProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index f308bce..80ef4b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -19,16 +19,22 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import com.google.protobuf.TextFormat;
public class QueueConfigurationsPBImpl extends QueueConfigurations {
- QueueConfigurationsProto proto =
- QueueConfigurationsProto.getDefaultInstance();
+ QueueConfigurationsProto proto = QueueConfigurationsProto
+ .getDefaultInstance();
QueueConfigurationsProto.Builder builder = null;
+ Resource configuredMinResource = null;
+ Resource configuredMaxResource = null;
+ Resource effMinResource = null;
+ Resource effMaxResource = null;
boolean viaProto = false;
public QueueConfigurationsPBImpl() {
@@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
}
public QueueConfigurationsProto getProto() {
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.effMinResource != null) {
+ builder
+ .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource));
+ }
+ if (this.effMaxResource != null) {
+ builder
+ .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
+ }
+ if (this.configuredMinResource != null) {
+ builder.setEffectiveMinCapacity(
+ convertToProtoFormat(this.configuredMinResource));
+ }
+ if (this.configuredMaxResource != null) {
+ builder.setEffectiveMaxCapacity(
+ convertToProtoFormat(this.configuredMaxResource));
+ }
+ }
+
@Override
public float getCapacity() {
QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
@@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
builder.setMaxAMPercentage(maxAMPercentage);
}
+ @Override
+ public Resource getEffectiveMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMinResource != null) {
+ return this.effMinResource;
+ }
+ if (!p.hasEffectiveMinCapacity()) {
+ return null;
+ }
+ this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity());
+ return this.effMinResource;
+ }
+
+ @Override
+ public void setEffectiveMinCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMinCapacity();
+ }
+ this.effMinResource = capacity;
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMaxResource != null) {
+ return this.effMaxResource;
+ }
+ if (!p.hasEffectiveMaxCapacity()) {
+ return null;
+ }
+ this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity());
+ return this.effMaxResource;
+ }
+
+ @Override
+ public void setEffectiveMaxCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMaxCapacity();
+ }
+ this.effMaxResource = capacity;
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ProtoUtils.convertToProtoFormat(t);
+ }
+
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = QueueConfigurationsProto.newBuilder(proto);
@@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
return false;
}
+ @Override
+ public Resource getConfiguredMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMinResource != null) {
+ return this.configuredMinResource;
+ }
+ if (!p.hasConfiguredMinCapacity()) {
+ return null;
+ }
+ this.configuredMinResource = convertFromProtoFormat(
+ p.getConfiguredMinCapacity());
+ return this.configuredMinResource;
+ }
+
+ @Override
+ public void setConfiguredMinCapacity(Resource configuredMinResource) {
+ maybeInitBuilder();
+ if (configuredMinResource == null) {
+ builder.clearConfiguredMinCapacity();
+ }
+ this.configuredMinResource = configuredMinResource;
+ }
+
+ @Override
+ public Resource getConfiguredMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMaxResource != null) {
+ return this.configuredMaxResource;
+ }
+ if (!p.hasConfiguredMaxCapacity()) {
+ return null;
+ }
+ this.configuredMaxResource = convertFromProtoFormat(
+ p.getConfiguredMaxCapacity());
+ return this.configuredMaxResource;
+ }
+
+ @Override
+ public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+ maybeInitBuilder();
+ if (configuredMaxResource == null) {
+ builder.clearConfiguredMaxCapacity();
+ }
+ this.configuredMaxResource = configuredMaxResource;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 16e4527..cc4acf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
- return Resources.createResource(
- roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
- roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ Resource ret = Resource.newInstance(r);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation rResourceInformation = r.getResourceInformation(i);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(i);
+ ResourceInformation tmp = ret.getResourceInformation(i);
+
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+
+ long value = rValue;
+ if (stepFactorValue != 0) {
+ value = roundDown(rValue, stepFactorValue);
+ }
+ tmp.setValue(value);
+ }
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
index 2e653fc..08b4d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage {
public void setEffectiveMaxResource(String label, Resource res) {
_set(label, ResourceType.EFF_MAX_RESOURCE, res);
}
-
- /*
- * Effective Minimum Resource
- */
- public Resource getEffectiveMinResourceUp() {
- return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public Resource getEffectiveMinResourceUp(String label) {
- return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public void setEffectiveMinResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- public void setEffectiveMinResourceUp(Resource res) {
- _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- /*
- * Effective Maximum Resource
- */
- public Resource getEffectiveMaxResourceUp() {
- return getEffectiveMaxResourceUp(NL);
- }
-
- public Resource getEffectiveMaxResourceUp(String label) {
- return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
- }
-
- public void setEffectiveMaxResourceUp(Resource res) {
- setEffectiveMaxResourceUp(NL, res);
- }
-
- public void setEffectiveMaxResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index dddac4d..9caf589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResource = conf.getMaximumResourceRequirement(label,
queuePath, resourceTypes);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
this.capacityConfigType = (!minResource.equals(Resources.none())
&& queueCapacities.getAbsoluteCapacity(label) == 0f)
@@ -481,12 +485,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public Resource getEffectiveCapacityUp(String label) {
- return Resources
- .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
- }
-
- @Override
public Resource getEffectiveCapacityDown(String label, Resource factor) {
return Resources.normalizeDown(resourceCalculator,
getQueueResourceQuotas().getEffectiveMinResource(label),
@@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue {
queueConfiguration.setMaxCapacity(maxCapacity);
queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+ queueConfiguration.setConfiguredMinCapacity(
+ queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+ queueConfiguration.setConfiguredMaxCapacity(
+ queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+ queueConfiguration.setEffectiveMinCapacity(
+ queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+ queueConfiguration.setEffectiveMaxCapacity(
+ queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
queueConfigurations.put(nodeLabel, queueConfiguration);
}
return queueConfigurations;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 2e29a71..624fcc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.security.AccessControlException;
@@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* <code>CSQueue</code> represents a node in the tree of
* hierarchical queues in the {@link CapacityScheduler}.
@@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return effective queue capacity
*/
Resource getEffectiveCapacity(String label);
- Resource getEffectiveCapacityUp(String label);
Resource getEffectiveCapacityDown(String label, Resource factor);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1460121..41ec4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index c770fac..5ab1494 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue {
queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ queueCapacities.setAbsoluteCapacity(label, 1.0f);
}
- // Total configured min resources of direct children of queue
+ // Total configured min resources of direct children of this given parent
+ // queue.
Resource configuredMinResources = Resource.newInstance(0L, 0);
for (CSQueue childQueue : getChildQueues()) {
Resources.addTo(configuredMinResources,
@@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue {
// present could also be taken from effective max resource of parent.
Resource parentMaxRes = queueResourceQuotas
.getConfiguredMaxResource(label);
- if (parentMaxRes.equals(Resources.none())) {
+ if (parent != null && parentMaxRes.equals(Resources.none())) {
parentMaxRes = parent.getQueueResourceQuotas()
.getEffectiveMaxResource(label);
}
@@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue {
parentMaxRes);
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.clone(effMaxResource));
+
+ // In cases where we still need to update some units based on
+ // percentage, we have to calculate percentage and update.
+ deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc,
+ childQueue);
} else {
childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
Resources.multiply(resourceByLabel,
@@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue {
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
.getAbsoluteMaximumCapacity(label)));
-
- childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
- childQueue.getQueueCapacities().getAbsoluteCapacity(label),
- minimumAllocation));
- childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc,
- resourceByLabel, childQueue.getQueueCapacities()
- .getAbsoluteMaximumCapacity(label),
- minimumAllocation));
}
if (LOG.isDebugEnabled()) {
@@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private void deriveCapacityFromAbsoluteConfigurations(String label,
+ Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
+
+ /*
+ * In case when queues are configured with absolute resources, it is better
+ * to update capacity/max-capacity etc w.r.t absolute resource as well. In
+ * case of computation, these values wont be used any more. However for
+ * metrics and UI, its better these values are pre-computed here itself.
+ */
+
+ // 1. Update capacity as a float based on parent's minResource
+ childQueue.getQueueCapacities().setCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMinResource(label),
+ getQueueResourceQuotas().getEffectiveMinResource(label)));
+
+ // 2. Update max-capacity as a float based on parent's maxResource
+ childQueue.getQueueCapacities().setMaximumCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label),
+ getQueueResourceQuotas().getEffectiveMaxResource(label)));
+
+ // 3. Update absolute capacity as a float based on parent's minResource and
+ // cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteCapacity(label,
+ (float) childQueue.getQueueCapacities().getCapacity()
+ / getQueueCapacities().getAbsoluteCapacity(label));
+
+ // 4. Update absolute max-capacity as a float based on parent's maxResource
+ // and cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
+ (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
+ / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index efc20e9..7287c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
+ Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 34594cf..a4e0096 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -909,6 +910,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
StringBuilder diagnosticMessage) {
LeafQueue queue = getCSLeafQueue();
QueueCapacities queueCapacities = queue.getQueueCapacities();
+ QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
diagnosticMessage.append(" Details : AM Partition = ");
diagnosticMessage.append(appAMNodePartitionName.isEmpty()
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
@@ -930,6 +932,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
* 100);
diagnosticMessage.append(" % ; ");
+ diagnosticMessage.append("Queue's capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's used capacity (absolute resource) = ");
+ diagnosticMessage
+ .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's max capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
}
/**
@@ -993,13 +1007,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
if (!calc.isInvalidDivisor(totalPartitionRes)) {
- float queueAbsMaxCapPerPartition =
- ((AbstractCSQueue) getQueue()).getQueueCapacities()
- .getAbsoluteCapacity(getAppAMNodePartitionName());
+ Resource effCap = ((AbstractCSQueue) getQueue())
+ .getEffectiveCapacity(getAppAMNodePartitionName());
float queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(),
- Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
- * 100;
+ report.getUsedResources(), effCap) * 100;
report.setQueueUsagePercentage(queueUsagePerc);
}
return report;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 74b4e79..3f68e07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView {
? new ResourceInfo(Resources.none())
: resourceUsages.getAmUsed();
ri.
- __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
- __("Configured Capacity:", percent(capacities.getCapacity() / 100)).
- __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)).
- __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
- __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
- __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
- __("Used Resources:", resourceUsages.getUsed().toString()).
- __("Configured Max Application Master Limit:", StringUtils.format("%.1f",
- capacities.getMaxAMLimitPercentage())).
- __("Max Application Master Resources:",
- resourceUsages.getAMLimit().toString()).
- __("Used Application Master Resources:",
- amUsed.toString()).
- __("Max Application Master Resources Per User:",
- userAMResourceLimit.toString());
+ __("Used Capacity:",
+ appendPercent(resourceUsages.getUsed().toString(),
+ capacities.getUsedCapacity() / 100))
+ .__("Configured Capacity:",
+ capacities.getConfiguredMinResource().toString())
+ .__("Configured Max Capacity:",
+ capacities.getConfiguredMaxResource().getResource()
+ .equals(Resources.none())
+ ? "unlimited"
+ : capacities.getConfiguredMaxResource().toString())
+ .__("Effective Capacity:",
+ appendPercent(capacities.getEffectiveMinResource().toString(),
+ capacities.getCapacity() / 100))
+ .__("Effective Max Capacity:",
+ appendPercent(capacities.getEffectiveMaxResource().toString(),
+ capacities.getMaxCapacity() / 100))
+ .__("Absolute Used Capacity:",
+ percent(capacities.getAbsoluteUsedCapacity() / 100))
+ .__("Absolute Configured Capacity:",
+ percent(capacities.getAbsoluteCapacity() / 100))
+ .__("Absolute Configured Max Capacity:",
+ percent(capacities.getAbsoluteMaxCapacity() / 100))
+ .__("Used Resources:", resourceUsages.getUsed().toString())
+ .__("Configured Max Application Master Limit:",
+ StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage()))
+ .__("Max Application Master Resources:",
+ resourceUsages.getAMLimit().toString())
+ .__("Used Application Master Resources:", amUsed.toString())
+ .__("Max Application Master Resources Per User:",
+ userAMResourceLimit.toString());
}
private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView {
return QueuesBlock.class;
}
+ static String appendPercent(String message, float f) {
+ return message + " (" + StringUtils.formatPercent(f, 1) + ")";
+ }
+
static String percent(float f) {
return StringUtils.formatPercent(f, 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 32e4ac5..ca42041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
max = 1f;
this.maxCapacity = max * 100;
- capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
+ capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
+ parent.getQueueResourceQuotas(), false);
queues = getQueues(parent);
health = new CapacitySchedulerHealthInfo(cs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 7dcdf58..343c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
}
@Override
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas);
}
public int getNumActiveApplications() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 86b2fea..d4de9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo {
Collections.sort(nodeLabels);
}
QueueCapacities qCapacities = q.getQueueCapacities();
- populateQueueCapacities(qCapacities);
+ QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas();
+ populateQueueCapacities(qCapacities, qResQuotas);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
@@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo {
resources = new ResourcesInfo(queueResourceUsage, false);
}
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities, false);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas,
+ false);
}
public float getCapacity() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
index 5e298f9..2a15502 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
@@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
/**
* This class represents queue capacities for a given partition
*/
@@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo {
private float absoluteUsedCapacity;
private float absoluteMaxCapacity = 100;
private float maxAMLimitPercentage;
+ private ResourceInfo configuredMinResource;
+ private ResourceInfo configuredMaxResource;
+ private ResourceInfo effectiveMinResource;
+ private ResourceInfo effectiveMaxResource;
public PartitionQueueCapacitiesInfo() {
}
public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
float usedCapacity, float maxCapacity, float absCapacity,
- float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) {
+ float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
+ Resource confMinRes, Resource confMaxRes, Resource effMinRes,
+ Resource effMaxRes) {
super();
this.partitionName = partitionName;
this.capacity = capacity;
@@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo {
this.absoluteUsedCapacity = absUsedCapacity;
this.absoluteMaxCapacity = absMaxCapacity;
this.maxAMLimitPercentage = maxAMLimitPercentage;
+ this.configuredMinResource = new ResourceInfo(confMinRes);
+ this.configuredMaxResource = new ResourceInfo(confMaxRes);
+ this.effectiveMinResource = new ResourceInfo(effMinRes);
+ this.effectiveMaxResource = new ResourceInfo(effMaxRes);
}
public float getCapacity() {
@@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo {
public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
this.maxAMLimitPercentage = maxAMLimitPercentage;
}
+
+ public ResourceInfo getConfiguredMinResource() {
+ return configuredMinResource;
+ }
+
+ public ResourceInfo getConfiguredMaxResource() {
+ if (configuredMaxResource.getResource().equals(Resources.none())) {
+ return null;
+ }
+ return configuredMaxResource;
+ }
+
+ public ResourceInfo getEffectiveMinResource() {
+ return effectiveMinResource;
+ }
+
+ public ResourceInfo getEffectiveMaxResource() {
+ return effectiveMaxResource;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
index 9a3e439..35c80d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
/**
@@ -39,6 +40,7 @@ public class QueueCapacitiesInfo {
}
public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas,
boolean considerAMUsage) {
if (capacities == null) {
return;
@@ -68,12 +70,17 @@ public class QueueCapacitiesInfo {
queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
absUsedCapacity, absMaxCapacity,
- considerAMUsage ? maxAMLimitPercentage : 0f));
+ considerAMUsage ? maxAMLimitPercentage : 0f,
+ resourceQuotas.getConfiguredMinResource(partitionName),
+ resourceQuotas.getConfiguredMaxResource(partitionName),
+ resourceQuotas.getEffectiveMinResource(partitionName),
+ resourceQuotas.getEffectiveMaxResource(partitionName)));
}
}
- public QueueCapacitiesInfo(QueueCapacities capacities) {
- this(capacities, true);
+ public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas) {
+ this(capacities, resourceQuotas, true);
}
public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index b386c18..38768e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
Map<String, ResourceInformation> riMap = new HashMap<>();
// Initialize mandatory resources
- riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
- riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
- riMap.put(RESOURCE_1, ResourceInformation
- .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
- Integer.MAX_VALUE));
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+ ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
@@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
- Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+ Assert.assertEquals(Resource.newInstance(1 * GB, 1),
leafQueue.getUsedResources());
RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
// Do node heartbeats 1 time and check container allocated.
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+ // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
Assert.assertEquals(
- TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+ TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
leafQueue.getUsedResources());
// Acquire container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 96f8970..30bff78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -1355,7 +1356,7 @@ public class TestLeafQueue {
// TODO, fix headroom in the future patch
assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
// User limit = 2G, 2 in use
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+ assertEquals(1*GB, app_1.getHeadroom().getMemorySize());
// the application is not yet active
// Again one to user_0 since he hasn't exceeded user limit yet
@@ -1366,8 +1367,8 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemorySize());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+ assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
@@ -3874,7 +3875,7 @@ public class TestLeafQueue {
// Queue "test" consumes 100% of the cluster, so its capacity and absolute
// capacity are both 1.0f.
- Queue queue = createQueue("test", null, 1.0f, 1.0f);
+ Queue queue = createQueue("test", null, 1.0f, 1.0f, res);
final String user = "user1";
FiCaSchedulerApp app =
new FiCaSchedulerApp(appAttId, user, queue,
@@ -3891,7 +3892,8 @@ public class TestLeafQueue {
// Queue "test2" is a child of root and its capacity is 50% of root. As a
// child of root, its absolute capaicty is also 50%.
- queue = createQueue("test2", null, 0.5f, 0.5f);
+ queue = createQueue("test2", null, 0.5f, 0.5f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 2));
app = new FiCaSchedulerApp(appAttId, user, queue,
queue.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3903,7 +3905,8 @@ public class TestLeafQueue {
// Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
// Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
- AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+ AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 4));
app = new FiCaSchedulerApp(appAttId, user, qChild,
qChild.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3922,7 +3925,7 @@ public class TestLeafQueue {
}
private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
- float absCap) {
+ float absCap, Resource res) {
CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
null, QueueState.RUNNING, null, "", null, false);
@@ -3934,6 +3937,13 @@ public class TestLeafQueue {
QueueCapacities qCaps = mock(QueueCapacities.class);
when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
when(queue.getQueueCapacities()).thenReturn(qCaps);
+ QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class);
+ when(qQuotas.getConfiguredMinResource(any())).thenReturn(res);
+ when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMinResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res);
+ when(queue.getQueueResourceQuotas()).thenReturn(qQuotas);
+ when(queue.getEffectiveCapacity(any())).thenReturn(res);
return queue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org