You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/01/20 19:49:14 UTC
[07/24] hadoop git commit: YARN-4304. AM max resource configuration
per partition to be displayed/updated correctly in UI and in various
partition related metrics. (Sunil G via wangda)
YARN-4304. AM max resource configuration per partition to be displayed/updated correctly in UI and in various partition related metrics. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b08ecf5c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b08ecf5c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b08ecf5c
Branch: refs/heads/HDFS-1312
Commit: b08ecf5c7589b055e93b2907413213f36097724d
Parents: 805a9ed
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Jan 18 11:07:15 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Jan 18 11:11:32 2016 +0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/capacity/AbstractCSQueue.java | 24 +++++
.../scheduler/capacity/CSQueue.java | 7 ++
.../scheduler/capacity/LeafQueue.java | 42 ++++++---
.../scheduler/capacity/UserInfo.java | 8 +-
.../webapp/CapacitySchedulerPage.java | 43 +++++++--
.../webapp/dao/CapacitySchedulerInfo.java | 2 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 23 ++++-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 12 ++-
.../dao/PartitionQueueCapacitiesInfo.java | 12 ++-
.../webapp/dao/PartitionResourceUsageInfo.java | 89 ------------------
.../webapp/dao/PartitionResourcesInfo.java | 99 ++++++++++++++++++++
.../webapp/dao/QueueCapacitiesInfo.java | 17 +++-
.../webapp/dao/ResourceUsageInfo.java | 79 ----------------
.../webapp/dao/ResourcesInfo.java | 81 ++++++++++++++++
.../capacity/TestApplicationLimits.java | 15 +--
.../scheduler/capacity/TestLeafQueue.java | 27 +++---
.../TestRMWebServicesForCSWithPartitions.java | 4 +-
18 files changed, 364 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8554dde..a9f4a84 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -706,6 +706,9 @@ Release 2.8.0 - UNRELEASED
YARN-4582. Label-related invalid resource request exception should be able to
properly handled by application. (Bibin A Chundatt via wangda)
+ YARN-4304. AM max resource configuration per partition to be displayed/updated
+ correctly in UI and in various partition related metrics. (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 62b1b56..39ca29b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -612,4 +613,27 @@ public abstract class AbstractCSQueue implements CSQueue {
// TODO add dummy implementation
return null;
}
+
+ @Override
+ public Set<String> getNodeLabelsForQueue() {
+ // if queue's label is *, queue can access any labels. Instead of
+ // considering all labels in cluster, only those labels which are
+ // use some resource of this queue can be considered.
+ Set<String> nodeLabels = new HashSet<String>();
+ if (this.getAccessibleNodeLabels() != null
+ && this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
+ nodeLabels.addAll(Sets.union(this.getQueueCapacities()
+ .getNodePartitionsSet(), this.getQueueResourceUsage()
+ .getNodePartitionsSet()));
+ } else {
+ nodeLabels.addAll(this.getAccessibleNodeLabels());
+ }
+
+ // Add NO_LABEL also to this list as NO_LABEL also can be granted with
+ // resource in many general cases.
+ if (!nodeLabels.contains(RMNodeLabelsManager.NO_LABEL)) {
+ nodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+ }
+ return nodeLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 12dc1cb..6ffba02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -332,4 +333,10 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public void decreaseContainer(Resource clusterResource,
SchedContainerChangeRequest decreaseRequest,
FiCaSchedulerApp app);
+
+ /**
+ * Get valid Node Labels for this queue
+ * @return valid node labels
+ */
+ public Set<String> getNodeLabelsForQueue();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index ff7d04f..9c6d8ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -528,10 +528,20 @@ public class LeafQueue extends AbstractCSQueue {
}
- public synchronized Resource getAMResourceLimit() {
- return getAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL);
+ public Resource getAMResourceLimit() {
+ return queueUsage.getAMLimit();
}
+ public Resource getAMResourceLimitPerPartition(String nodePartition) {
+ return queueUsage.getAMLimit(nodePartition);
+ }
+
+ public synchronized Resource calculateAndGetAMResourceLimit() {
+ return calculateAndGetAMResourceLimitPerPartition(
+ RMNodeLabelsManager.NO_LABEL);
+ }
+
+ @VisibleForTesting
public synchronized Resource getUserAMResourceLimit() {
return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL);
}
@@ -552,13 +562,17 @@ public class LeafQueue extends AbstractCSQueue {
labelManager.getResourceByLabel(nodePartition, lastClusterResource),
queueCapacities.getAbsoluteCapacity(nodePartition), minimumAllocation);
- return Resources.multiplyAndNormalizeUp(resourceCalculator,
+ Resource userAMLimit = Resources.multiplyAndNormalizeUp(resourceCalculator,
queuePartitionResource,
queueCapacities.getMaxAMResourcePercentage(nodePartition)
* effectiveUserLimit * userLimitFactor, minimumAllocation);
+ return Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
+ userAMLimit, getAMResourceLimitPerPartition(nodePartition))
+ ? userAMLimit
+ : getAMResourceLimitPerPartition(nodePartition);
}
- public synchronized Resource getAMResourceLimitPerPartition(
+ public synchronized Resource calculateAndGetAMResourceLimitPerPartition(
String nodePartition) {
/*
* For non-labeled partition, get the max value from resources currently
@@ -601,20 +615,26 @@ public class LeafQueue extends AbstractCSQueue {
private synchronized void activateApplications() {
// limit of allowed resource usage for application masters
- Map<String, Resource> amPartitionLimit = new HashMap<String, Resource>();
Map<String, Resource> userAmPartitionLimit =
new HashMap<String, Resource>();
+ // AM Resource Limit for accessible labels can be pre-calculated.
+ // This will help in updating AMResourceLimit for all labels when queue
+ // is initialized for the first time (when no applications are present).
+ for (String nodePartition : getNodeLabelsForQueue()) {
+ calculateAndGetAMResourceLimitPerPartition(nodePartition);
+ }
+
activateApplications(getPendingAppsOrderingPolicyRecovery()
- .getAssignmentIterator(), amPartitionLimit, userAmPartitionLimit);
+ .getAssignmentIterator(), userAmPartitionLimit);
activateApplications(
getPendingAppsOrderingPolicy().getAssignmentIterator(),
- amPartitionLimit, userAmPartitionLimit);
+ userAmPartitionLimit);
}
private synchronized void activateApplications(
- Iterator<FiCaSchedulerApp> fsApp, Map<String, Resource> amPartitionLimit,
+ Iterator<FiCaSchedulerApp> fsApp,
Map<String, Resource> userAmPartitionLimit) {
while (fsApp.hasNext()) {
FiCaSchedulerApp application = fsApp.next();
@@ -624,11 +644,10 @@ public class LeafQueue extends AbstractCSQueue {
// and calculate max-am resource limit for this partition.
String partitionName = application.getAppAMNodePartitionName();
- Resource amLimit = amPartitionLimit.get(partitionName);
+ Resource amLimit = getAMResourceLimitPerPartition(partitionName);
// Verify whether we already calculated am-limit for this label.
if (amLimit == null) {
- amLimit = getAMResourceLimitPerPartition(partitionName);
- amPartitionLimit.put(partitionName, amLimit);
+ amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
}
// Check am resource limit.
Resource amIfStarted = Resources.add(
@@ -705,6 +724,7 @@ public class LeafQueue extends AbstractCSQueue {
application.getAMResource(partitionName));
user.getResourceUsage().incAMUsed(partitionName,
application.getAMResource(partitionName));
+ user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
metrics.incAMUsed(application.getUser(),
application.getAMResource(partitionName));
metrics.setAMResouceLimitForUser(application.getUser(), userAMLimit);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java
index f51187b..ff9d304 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java
@@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceUsageInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourcesInfo;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -36,7 +36,7 @@ public class UserInfo {
protected int numActiveApplications;
protected ResourceInfo AMResourceUsed;
protected ResourceInfo userResourceLimit;
- protected ResourceUsageInfo resources;
+ protected ResourcesInfo resources;
UserInfo() {}
@@ -48,7 +48,7 @@ public class UserInfo {
this.numPendingApplications = pendingApps;
this.AMResourceUsed = new ResourceInfo(amResUsed);
this.userResourceLimit = new ResourceInfo(resourceLimit);
- this.resources = new ResourceUsageInfo(resourceUsage);
+ this.resources = new ResourcesInfo(resourceUsage);
}
public String getUsername() {
@@ -75,7 +75,7 @@ public class UserInfo {
return userResourceLimit;
}
- public ResourceUsageInfo getResourceUsageInfo() {
+ public ResourcesInfo getResourceUsageInfo() {
return resources;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index b4114de..033152a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -39,10 +39,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionQueueCapacitiesInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionResourceUsageInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionResourcesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
@@ -134,8 +135,23 @@ class CapacitySchedulerPage extends RmView {
private void renderQueueCapacityInfo(ResponseInfo ri, String label) {
PartitionQueueCapacitiesInfo capacities =
lqinfo.getCapacities().getPartitionQueueCapacitiesInfo(label);
- PartitionResourceUsageInfo resourceUsages =
+ PartitionResourcesInfo resourceUsages =
lqinfo.getResources().getPartitionResourceUsageInfo(label);
+
+ // Get UserInfo from first user to calculate AM Resource Limit per user.
+ ResourceInfo userAMResourceLimit = null;
+ ArrayList<UserInfo> usersList = lqinfo.getUsers().getUsersList();
+ if (usersList.isEmpty()) {
+ // If no users are present, consider AM Limit for that queue.
+ userAMResourceLimit = resourceUsages.getAMLimit();
+ } else {
+ userAMResourceLimit = usersList.get(0)
+ .getResourceUsageInfo().getPartitionResourceUsageInfo(label)
+ .getAMLimit();
+ }
+ ResourceInfo amUsed = (resourceUsages.getAmUsed() == null)
+ ? new ResourceInfo(Resources.none())
+ : resourceUsages.getAmUsed();
ri.
_("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
_("Configured Capacity:", percent(capacities.getCapacity() / 100)).
@@ -143,7 +159,15 @@ class CapacitySchedulerPage extends RmView {
_("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
_("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
_("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
- _("Used Resources:", resourceUsages.getUsed().toString());
+ _("Used Resources:", resourceUsages.getUsed().toString()).
+ _("Configured Max Application Master Limit:", StringUtils.format("%.1f",
+ capacities.getMaxAMLimitPercentage())).
+ _("Max Application Master Resources:",
+ resourceUsages.getAMLimit().toString()).
+ _("Used Application Master Resources:",
+ amUsed.toString()).
+ _("Max Application Master Resources Per User:",
+ userAMResourceLimit.toString());
}
private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -153,9 +177,6 @@ class CapacitySchedulerPage extends RmView {
_("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
_("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
_("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
- _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()).
- _("Used Application Master Resources:", lqinfo.getUsedAMResource().toString()).
- _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
_("Configured User Limit Factor:", lqinfo.getUserLimitFactor()).
_("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())).
@@ -197,15 +218,21 @@ class CapacitySchedulerPage extends RmView {
ArrayList<UserInfo> users = lqinfo.getUsers().getUsersList();
for (UserInfo userInfo : users) {
ResourceInfo resourcesUsed = userInfo.getResourcesUsed();
+ PartitionResourcesInfo resourceUsages = lqinfo
+ .getResources()
+ .getPartitionResourceUsageInfo((nodeLabel == null) ? "" : nodeLabel);
if (nodeLabel != null) {
resourcesUsed = userInfo.getResourceUsageInfo()
.getPartitionResourceUsageInfo(nodeLabel).getUsed();
}
+ ResourceInfo amUsed = (resourceUsages.getAmUsed() == null)
+ ? new ResourceInfo(Resources.none())
+ : resourceUsages.getAmUsed();
tbody.tr().td(userInfo.getUsername())
.td(userInfo.getUserResourceLimit().toString())
.td(resourcesUsed.toString())
- .td(lqinfo.getUserAMResourceLimit().toString())
- .td(userInfo.getAMResourcesUsed().toString())
+ .td(resourceUsages.getAMLimit().toString())
+ .td(amUsed.toString())
.td(Integer.toString(userInfo.getNumActiveApplications()))
.td(Integer.toString(userInfo.getNumPendingApplications()))._();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index db22ada..f6332c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -56,7 +56,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
max = 1f;
this.maxCapacity = max * 100;
- capacities = new QueueCapacitiesInfo(parent.getQueueCapacities());
+ capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
queues = getQueues(parent);
health = new CapacitySchedulerHealthInfo(cs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 54fd6eb..e0ac56f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -17,13 +17,18 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import java.util.ArrayList;
+
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -62,16 +67,30 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userLimitFactor = q.getUserLimitFactor();
AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
- userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
preemptionDisabled = q.getPreemptionDisabled();
orderingPolicyInfo = q.getOrderingPolicy().getInfo();
defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
defaultPriority = q.getDefaultApplicationPriority().getPriority();
+ ArrayList<UserInfo> usersList = users.getUsersList();
+ if (usersList.isEmpty()) {
+ // If no users are present, consider AM Limit for that queue.
+ userAMResourceLimit = resources.getPartitionResourceUsageInfo(
+ RMNodeLabelsManager.NO_LABEL).getAMLimit();
+ } else {
+ userAMResourceLimit = usersList.get(0).getResourceUsageInfo()
+ .getPartitionResourceUsageInfo(RMNodeLabelsManager.NO_LABEL)
+ .getAMLimit();
+ }
}
@Override
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
- resources = new ResourceUsageInfo(queueResourceUsage);
+ resources = new ResourcesInfo(queueResourceUsage);
+ }
+
+ @Override
+ protected void populateQueueCapacities(QueueCapacities qCapacities) {
+ capacities = new QueueCapacitiesInfo(qCapacities);
}
public int getNumActiveApplications() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 1b6347d..22705cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -61,7 +61,7 @@ public class CapacitySchedulerQueueInfo {
protected long reservedContainers;
protected long pendingContainers;
protected QueueCapacitiesInfo capacities;
- protected ResourceUsageInfo resources;
+ protected ResourcesInfo resources;
CapacitySchedulerQueueInfo() {
};
@@ -101,14 +101,18 @@ public class CapacitySchedulerQueueInfo {
Collections.sort(nodeLabels);
}
QueueCapacities qCapacities = q.getQueueCapacities();
- capacities = new QueueCapacitiesInfo(qCapacities);
+ populateQueueCapacities(qCapacities);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
- resources = new ResourceUsageInfo(queueResourceUsage, false);
+ resources = new ResourcesInfo(queueResourceUsage, false);
+ }
+
+ protected void populateQueueCapacities(QueueCapacities qCapacities) {
+ capacities = new QueueCapacitiesInfo(qCapacities, false);
}
public float getCapacity() {
@@ -193,7 +197,7 @@ public class CapacitySchedulerQueueInfo {
return capacities;
}
- public ResourceUsageInfo getResources() {
+ public ResourcesInfo getResources() {
return resources;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
index 8f3975b..9c73bf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
@@ -35,13 +35,14 @@ public class PartitionQueueCapacitiesInfo {
private float absoluteCapacity;
private float absoluteUsedCapacity;
private float absoluteMaxCapacity = 100;
+ private Float maxAMLimitPercentage;
public PartitionQueueCapacitiesInfo() {
}
public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
float usedCapacity, float maxCapacity, float absCapacity,
- float absUsedCapacity, float absMaxCapacity) {
+ float absUsedCapacity, float absMaxCapacity, Float maxAMLimitPercentage) {
super();
this.partitionName = partitionName;
this.capacity = capacity;
@@ -50,6 +51,7 @@ public class PartitionQueueCapacitiesInfo {
this.absoluteCapacity = absCapacity;
this.absoluteUsedCapacity = absUsedCapacity;
this.absoluteMaxCapacity = absMaxCapacity;
+ this.maxAMLimitPercentage = maxAMLimitPercentage;
}
public float getCapacity() {
@@ -107,4 +109,12 @@ public class PartitionQueueCapacitiesInfo {
public void setAbsoluteMaxCapacity(float absoluteMaxCapacity) {
this.absoluteMaxCapacity = absoluteMaxCapacity;
}
+
+ public float getMaxAMLimitPercentage() {
+ return maxAMLimitPercentage;
+ }
+
+ public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
+ this.maxAMLimitPercentage = maxAMLimitPercentage;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourceUsageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourceUsageInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourceUsageInfo.java
deleted file mode 100644
index c202228..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourceUsageInfo.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * This class represents queue/user resource usage info for a given partition
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class PartitionResourceUsageInfo {
- private String partitionName;
- private ResourceInfo used = new ResourceInfo();
- private ResourceInfo reserved;
- private ResourceInfo pending;
- private ResourceInfo amUsed;
-
- public PartitionResourceUsageInfo() {
- }
-
- public PartitionResourceUsageInfo(String partitionName, ResourceInfo used,
- ResourceInfo reserved, ResourceInfo pending,
- ResourceInfo amResourceUsed) {
- super();
- this.partitionName = partitionName;
- this.used = used;
- this.reserved = reserved;
- this.pending = pending;
- this.amUsed = amResourceUsed;
- }
-
- public String getPartitionName() {
- return partitionName;
- }
-
- public void setPartitionName(String partitionName) {
- this.partitionName = partitionName;
- }
-
- public ResourceInfo getUsed() {
- return used;
- }
-
- public void setUsed(ResourceInfo used) {
- this.used = used;
- }
-
- public ResourceInfo getReserved() {
- return reserved;
- }
-
- public void setReserved(ResourceInfo reserved) {
- this.reserved = reserved;
- }
-
- public ResourceInfo getPending() {
- return pending;
- }
-
- public void setPending(ResourceInfo pending) {
- this.pending = pending;
- }
-
- public ResourceInfo getAmUsed() {
- return amUsed;
- }
-
- public void setAmUsed(ResourceInfo amResourceUsed) {
- this.amUsed = amResourceUsed;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourcesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourcesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourcesInfo.java
new file mode 100644
index 0000000..8b43f5d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionResourcesInfo.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class represents queue/user resource usage info for a given partition
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PartitionResourcesInfo {
+ private String partitionName;
+ private ResourceInfo used = new ResourceInfo();
+ private ResourceInfo reserved;
+ private ResourceInfo pending;
+ private ResourceInfo amUsed;
+ private ResourceInfo amLimit = new ResourceInfo();
+
+ public PartitionResourcesInfo() {
+ }
+
+ public PartitionResourcesInfo(String partitionName, ResourceInfo used,
+ ResourceInfo reserved, ResourceInfo pending,
+ ResourceInfo amResourceUsed, ResourceInfo amResourceLimit) {
+ super();
+ this.partitionName = partitionName;
+ this.used = used;
+ this.reserved = reserved;
+ this.pending = pending;
+ this.amUsed = amResourceUsed;
+ this.amLimit = amResourceLimit;
+ }
+
+ public String getPartitionName() {
+ return partitionName;
+ }
+
+ public void setPartitionName(String partitionName) {
+ this.partitionName = partitionName;
+ }
+
+ public ResourceInfo getUsed() {
+ return used;
+ }
+
+ public void setUsed(ResourceInfo used) {
+ this.used = used;
+ }
+
+ public ResourceInfo getReserved() {
+ return reserved;
+ }
+
+ public void setReserved(ResourceInfo reserved) {
+ this.reserved = reserved;
+ }
+
+ public ResourceInfo getPending() {
+ return pending;
+ }
+
+ public void setPending(ResourceInfo pending) {
+ this.pending = pending;
+ }
+
+ public ResourceInfo getAmUsed() {
+ return amUsed;
+ }
+
+ public void setAmUsed(ResourceInfo amResourceUsed) {
+ this.amUsed = amResourceUsed;
+ }
+
+ public ResourceInfo getAMLimit() {
+ return amLimit;
+ }
+
+ public void setAMLimit(ResourceInfo amLimit) {
+ this.amLimit = amLimit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
index 65cf0cb..1f0aa86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
@@ -38,7 +38,8 @@ public class QueueCapacitiesInfo {
public QueueCapacitiesInfo() {
}
- public QueueCapacitiesInfo(QueueCapacities capacities) {
+ public QueueCapacitiesInfo(QueueCapacities capacities,
+ boolean considerAMUsage) {
if (capacities == null) {
return;
}
@@ -48,6 +49,7 @@ public class QueueCapacitiesInfo {
float absCapacity;
float absUsedCapacity;
float absMaxCapacity;
+ Float maxAMLimitPercentage;
for (String partitionName : capacities.getExistingNodeLabels()) {
usedCapacity = capacities.getUsedCapacity(partitionName) * 100;
capacity = capacities.getCapacity(partitionName) * 100;
@@ -58,15 +60,22 @@ public class QueueCapacitiesInfo {
.cap(capacities.getAbsoluteUsedCapacity(partitionName), 0f, 1f) * 100;
absMaxCapacity = CapacitySchedulerQueueInfo.cap(
capacities.getAbsoluteMaximumCapacity(partitionName), 0f, 1f) * 100;
+ maxAMLimitPercentage = capacities
+ .getMaxAMResourcePercentage(partitionName) * 100;
if (maxCapacity < CapacitySchedulerQueueInfo.EPSILON || maxCapacity > 1f)
maxCapacity = 1f;
maxCapacity = maxCapacity * 100;
- queueCapacitiesByPartition.add(
- new PartitionQueueCapacitiesInfo(partitionName, capacity, usedCapacity,
- maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity));
+ queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
+ partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
+ absUsedCapacity, absMaxCapacity,
+ considerAMUsage ? maxAMLimitPercentage : null));
}
}
+ public QueueCapacitiesInfo(QueueCapacities capacities) {
+ this(capacities, true);
+ }
+
public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {
queueCapacitiesByPartition.add(partitionQueueCapacitiesInfo);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUsageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUsageInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUsageInfo.java
deleted file mode 100644
index 95153d5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUsageInfo.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
-
-/**
- * DAO which wraps PartitionResourceUsageInfo applicable for a queue/user
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ResourceUsageInfo {
- List<PartitionResourceUsageInfo> resourceUsagesByPartition =
- new ArrayList<>();
-
- public ResourceUsageInfo() {
- }
-
- public ResourceUsageInfo(ResourceUsage resourceUsage,
- boolean considerAMUsage) {
- if (resourceUsage == null) {
- return;
- }
- for (String partitionName : resourceUsage.getNodePartitionsSet()) {
- resourceUsagesByPartition.add(new PartitionResourceUsageInfo(
- partitionName, new ResourceInfo(resourceUsage.getUsed(partitionName)),
- new ResourceInfo(resourceUsage.getReserved(partitionName)),
- new ResourceInfo(resourceUsage.getPending(partitionName)),
- considerAMUsage
- ? new ResourceInfo(resourceUsage.getAMUsed(partitionName))
- : null));
- }
- }
-
- public ResourceUsageInfo(ResourceUsage resourceUsage) {
- this(resourceUsage, true);
- }
-
- public List<PartitionResourceUsageInfo> getPartitionResourceUsages() {
- return resourceUsagesByPartition;
- }
-
- public void setPartitionResourceUsages(
- List<PartitionResourceUsageInfo> resources) {
- this.resourceUsagesByPartition = resources;
- }
-
- public PartitionResourceUsageInfo getPartitionResourceUsageInfo(
- String partitionName) {
- for (PartitionResourceUsageInfo partitionResourceUsageInfo : resourceUsagesByPartition) {
- if (partitionResourceUsageInfo.getPartitionName().equals(partitionName)) {
- return partitionResourceUsageInfo;
- }
- }
- return new PartitionResourceUsageInfo();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
new file mode 100644
index 0000000..4f1e1c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+
+/**
+ * DAO which wraps PartitionResourceUsageInfo applicable for a queue/user
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourcesInfo {
+ List<PartitionResourcesInfo> resourceUsagesByPartition =
+ new ArrayList<>();
+
+ public ResourcesInfo() {
+ }
+
+ public ResourcesInfo(ResourceUsage resourceUsage,
+ boolean considerAMUsage) {
+ if (resourceUsage == null) {
+ return;
+ }
+ for (String partitionName : resourceUsage.getNodePartitionsSet()) {
+ resourceUsagesByPartition.add(new PartitionResourcesInfo(partitionName,
+ new ResourceInfo(resourceUsage.getUsed(partitionName)),
+ new ResourceInfo(resourceUsage.getReserved(partitionName)),
+ new ResourceInfo(resourceUsage.getPending(partitionName)),
+ considerAMUsage ? new ResourceInfo(resourceUsage
+ .getAMUsed(partitionName)) : null,
+ considerAMUsage ? new ResourceInfo(resourceUsage
+ .getAMLimit(partitionName)) : null));
+ }
+ }
+
+ public ResourcesInfo(ResourceUsage resourceUsage) {
+ this(resourceUsage, true);
+ }
+
+ public List<PartitionResourcesInfo> getPartitionResourceUsages() {
+ return resourceUsagesByPartition;
+ }
+
+ public void setPartitionResourceUsages(
+ List<PartitionResourcesInfo> resources) {
+ this.resourceUsagesByPartition = resources;
+ }
+
+ public PartitionResourcesInfo getPartitionResourceUsageInfo(
+ String partitionName) {
+ for (PartitionResourcesInfo partitionResourceUsageInfo :
+ resourceUsagesByPartition) {
+ if (partitionResourceUsageInfo.getPartitionName().equals(partitionName)) {
+ return partitionResourceUsageInfo;
+ }
+ }
+ return new PartitionResourcesInfo();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 3e65013..0b32676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -181,7 +181,8 @@ public class TestApplicationLimits {
ActiveUsersManager activeUsersManager = mock(ActiveUsersManager.class);
when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
- assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(8 * GB, 1),
+ queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(4 * GB, 1),
queue.getUserAMResourceLimit());
@@ -282,8 +283,7 @@ public class TestApplicationLimits {
queue.getUserAMResourceLimit());
Resource amResourceLimit = Resource.newInstance(160 * GB, 1);
- assertEquals(queue.getAMResourceLimit(), amResourceLimit);
- assertEquals(queue.getAMResourceLimit(), amResourceLimit);
+ assertEquals(queue.calculateAndGetAMResourceLimit(), amResourceLimit);
assertEquals(queue.getUserAMResourceLimit(),
Resource.newInstance(80*GB, 1));
@@ -303,7 +303,8 @@ public class TestApplicationLimits {
root.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
- assertEquals(queue.getAMResourceLimit(), Resource.newInstance(192*GB, 1));
+ assertEquals(queue.calculateAndGetAMResourceLimit(),
+ Resource.newInstance(192 * GB, 1));
assertEquals(queue.getUserAMResourceLimit(),
Resource.newInstance(96*GB, 1));
@@ -353,7 +354,8 @@ public class TestApplicationLimits {
queue.getQueuePath())
);
- assertEquals(queue.getAMResourceLimit(), Resource.newInstance(800*GB, 1));
+ assertEquals(queue.calculateAndGetAMResourceLimit(),
+ Resource.newInstance(800 * GB, 1));
assertEquals(queue.getUserAMResourceLimit(),
Resource.newInstance(400*GB, 1));
@@ -383,7 +385,8 @@ public class TestApplicationLimits {
final String user_1 = "user_1";
final String user_2 = "user_2";
- assertEquals(Resource.newInstance(16 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(16 * GB, 1),
+ queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(8 * GB, 1),
queue.getUserAMResourceLimit());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index d4b8dcd..1922a35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -263,18 +263,18 @@ public class TestLeafQueue {
assertEquals(0.1, c.getMaximumCapacity(), epsilon);
assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
- //Verify the value for getAMResourceLimit for queues with < .1 maxcap
- Resource clusterResource = Resource.newInstance(50 * GB, 50);
-
+ // Verify the value for getAMResourceLimit for queues with < .1 maxcap
+ Resource clusterResource = Resource.newInstance(50 * GB, 50);
+
a.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
- assertEquals(Resource.newInstance(1 * GB, 1),
- a.getAMResourceLimit());
-
- b.updateClusterResource(clusterResource,
+ assertEquals(Resource.newInstance(1 * GB, 1),
+ a.calculateAndGetAMResourceLimit());
+
+ b.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
- assertEquals(Resource.newInstance(5 * GB, 1),
- b.getAMResourceLimit());
+ assertEquals(Resource.newInstance(5 * GB, 1),
+ b.calculateAndGetAMResourceLimit());
}
@Test
@@ -2323,21 +2323,24 @@ public class TestLeafQueue {
csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
LeafQueue a = new LeafQueue(csContext, A, root, null);
assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
+ assertEquals(a.calculateAndGetAMResourceLimit(),
+ Resources.createResource(160 * GB, 1));
csConf.setFloat(CapacitySchedulerConfiguration.
MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
LeafQueue newA = new LeafQueue(csContext, A, root, null);
a.reinitialize(newA, clusterResource);
assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));
+ assertEquals(a.calculateAndGetAMResourceLimit(),
+ Resources.createResource(320 * GB, 1));
Resource newClusterResource = Resources.createResource(100 * 20 * GB,
100 * 32);
a.updateClusterResource(newClusterResource,
new ResourceLimits(newClusterResource));
// 100 * 20 * 0.2 = 400
- assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
+ assertEquals(a.calculateAndGetAMResourceLimit(),
+ Resources.createResource(400 * GB, 1));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
index aec537d..8087e33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
@@ -501,13 +501,13 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
partitionInfo = partitionsCapsArray.getJSONObject(0);
partitionName = partitionInfo.getString("partitionName");
verifyPartitionCapacityInfoJson(partitionInfo, 30, 0, 50, 30, 0, 50);
- assertEquals("incorrect number of elements", 5,
+ assertEquals("incorrect number of elements", 6,
partitionsResourcesArray.getJSONObject(0).length());
break;
case QUEUE_B:
assertEquals("Invalid default Label expression", LABEL_LX,
queueJson.getString("defaultNodeLabelExpression"));
- assertEquals("incorrect number of elements", 5,
+ assertEquals("incorrect number of elements", 6,
partitionsResourcesArray.getJSONObject(0).length());
verifyAccesibleNodeLabels(queueJson, ImmutableSet.of(LABEL_LX));
assertEquals("incorrect number of partitions", 2,