You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/07/28 20:49:10 UTC
[hadoop] branch branch-3.1 updated: YARN-10343. Legacy RM UI should
include labeled metrics for allocated, total,
and reserved resources. Contributed by Eric Payne
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b0edec2 YARN-10343. Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. Contributed by Eric Payne
b0edec2 is described below
commit b0edec29182206ad84da673671c56902e4d390cd
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Tue Jul 28 13:44:17 2020 -0700
YARN-10343. Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. Contributed by Eric Payne
(cherry picked from commit ffb920de2a60558a9863afb31b566b8c688a8263)
---
.../resourcemanager/scheduler/ResourceUsage.java | 4 ++
.../webapp/MetricsOverviewTable.java | 49 ++++++++++++++++++----
.../webapp/dao/ClusterMetricsInfo.java | 26 ++++++++++++
.../resourcemanager/webapp/TestRMWebServices.java | 2 +-
4 files changed, 72 insertions(+), 9 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 37958de..ea8a47a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -200,6 +200,10 @@ public class ResourceUsage extends AbstractResourceUsage {
return _getAll(ResourceType.USED);
}
+ public Resource getAllReserved() {
+ return _getAll(ResourceType.RESERVED);
+ }
+
// Cache Used
public Resource getCachedUsed() {
return _get(NL, ResourceType.CACHED_USED);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
index 806b636..fbaeafd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
@@ -60,7 +61,38 @@ public class MetricsOverviewTable extends HtmlBlock {
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm);
DIV<Hamlet> div = html.div().$class("metrics");
-
+
+ long usedMemoryBytes = 0;
+ long totalMemoryBytes = 0;
+ long reservedMemoryBytes = 0;
+ long usedVCores = 0;
+ long totalVCores = 0;
+ long reservedVCores = 0;
+ if (clusterMetrics.getCrossPartitionMetricsAvailable()) {
+ ResourceInfo usedAllPartitions =
+ clusterMetrics.getTotalUsedResourcesAcrossPartition();
+ ResourceInfo totalAllPartitions =
+ clusterMetrics.getTotalClusterResourcesAcrossPartition();
+ ResourceInfo reservedAllPartitions =
+ clusterMetrics.getTotalReservedResourcesAcrossPartition();
+ usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB;
+ totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB;
+ reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB;
+ usedVCores = usedAllPartitions.getvCores();
+ totalVCores = totalAllPartitions.getvCores();
+ reservedVCores = reservedAllPartitions.getvCores();
+ // getTotalUsedResourcesAcrossPartition includes reserved resources.
+ usedMemoryBytes -= reservedMemoryBytes;
+ usedVCores -= reservedVCores;
+ } else {
+ usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB;
+ totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB;
+ reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB;
+ usedVCores = clusterMetrics.getAllocatedVirtualCores();
+ totalVCores = clusterMetrics.getTotalVirtualCores();
+ reservedVCores = clusterMetrics.getReservedVirtualCores();
+ }
+
div.h3("Cluster Metrics").
table("#metricsoverview").
thead().$class("ui-widget-header").
@@ -89,13 +121,14 @@ public class MetricsOverviewTable extends HtmlBlock {
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
)
).
- td(String.valueOf(clusterMetrics.getContainersAllocated())).
- td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
- td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
- td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
- td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
- td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
- td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
+ td(String.valueOf(
+ clusterMetrics.getTotalAllocatedContainersAcrossPartition())).
+ td(StringUtils.byteDesc(usedMemoryBytes)).
+ td(StringUtils.byteDesc(totalMemoryBytes)).
+ td(StringUtils.byteDesc(reservedMemoryBytes)).
+ td(String.valueOf(usedVCores)).
+ td(String.valueOf(totalVCores)).
+ td(String.valueOf(reservedVCores)).
__().
__().__();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 69d88aa..30954f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
@XmlRootElement(name = "clusterMetrics")
@XmlAccessorType(XmlAccessType.FIELD)
@@ -67,6 +68,14 @@ public class ClusterMetricsInfo {
// Total registered resources of the cluster, including all partitions
private ResourceInfo totalClusterResourcesAcrossPartition;
+ // Total reserved resources of the cluster, including all partitions.
+ private ResourceInfo totalReservedResourcesAcrossPartition;
+
+ // Total allocated containers across all partitions.
+ private int totalAllocatedContainersAcrossPartition;
+
+ private boolean crossPartitionMetricsAvailable = false;
+
public ClusterMetricsInfo() {
} // JAXB needs this
@@ -111,6 +120,11 @@ public class ClusterMetricsInfo {
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
totalClusterResourcesAcrossPartition = new ResourceInfo(
cs.getClusterResource());
+ totalReservedResourcesAcrossPartition = new ResourceInfo(
+ cs.getRootQueue().getQueueResourceUsage().getAllReserved());
+ totalAllocatedContainersAcrossPartition =
+ ((ParentQueue) cs.getRootQueue()).getNumContainers();
+ crossPartitionMetricsAvailable = true;
}
} else {
this.totalMB = availableMB + allocatedMB;
@@ -334,4 +348,16 @@ public class ClusterMetricsInfo {
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
return totalClusterResourcesAcrossPartition;
}
+
+ public ResourceInfo getTotalReservedResourcesAcrossPartition() {
+ return totalReservedResourcesAcrossPartition;
+ }
+
+ public int getTotalAllocatedContainersAcrossPartition() {
+ return totalAllocatedContainersAcrossPartition;
+ }
+
+ public boolean getCrossPartitionMetricsAvailable() {
+ return crossPartitionMetricsAvailable;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 5e3fab9..4498427 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -461,7 +461,7 @@ public class TestRMWebServices extends JerseyTestBase {
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
- assertEquals("incorrect number of elements", 25, clusterinfo.length());
+ assertEquals("incorrect number of elements", 27, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org