You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/08/08 00:45:36 UTC
[hadoop] branch branch-2.10 updated: YARN-10251. Show extended
resources on legacy RM UI. Contributed by Eric Payne
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 865828a YARN-10251. Show extended resources on legacy RM UI. Contributed by Eric Payne
865828a is described below
commit 865828ae6354872a6cbd6e663ed43f261f643ffd
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Fri Aug 7 17:45:04 2020 -0700
YARN-10251. Show extended resources on legacy RM UI. Contributed by Eric Payne
---
.../hadoop/yarn/server/webapp/WebPageUtils.java | 4 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 22 +++++++
.../webapp/MetricsOverviewTable.java | 75 ++++++++++------------
.../server/resourcemanager/webapp/NodesPage.java | 22 ++++++-
.../server/resourcemanager/webapp/RMAppsBlock.java | 10 +++
.../resourcemanager/webapp/TestNodesPage.java | 4 +-
6 files changed, 91 insertions(+), 46 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 653a741..285ffca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -56,7 +56,7 @@ public class WebPageUtils {
.append(", 'mRender': renderHadoopDate }");
if (isResourceManager) {
sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }");
- progressIndex = "[18]";
+ progressIndex = "[20]";
} else if (isFairSchedulerPage) {
sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }");
progressIndex = "[16]";
@@ -103,4 +103,4 @@ public class WebPageUtils {
.toString();
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index 27469a8..b52a9ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -30,8 +30,10 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@Public
@Evolving
@@ -63,8 +65,10 @@ public class AppInfo {
protected int priority;
private long allocatedCpuVcores;
private long allocatedMemoryMB;
+ private long allocatedGpus;
private long reservedCpuVcores;
private long reservedMemoryMB;
+ private long reservedGpus;
protected boolean unmanagedApplication;
private String appNodeLabelExpression;
private String amNodeLabelExpression;
@@ -110,6 +114,16 @@ public class AppInfo {
reservedCpuVcores = usageReport.getReservedResources().
getVirtualCores();
reservedMemoryMB = usageReport.getReservedResources().getMemorySize();
+ Integer gpuIndex = ResourceUtils.getResourceTypeIndex()
+ .get(ResourceInformation.GPU_URI);
+ allocatedGpus = -1;
+ reservedGpus = -1;
+ if (gpuIndex != null) {
+ allocatedGpus = usageReport.getUsedResources()
+ .getResourceValue(ResourceInformation.GPU_URI);
+ reservedGpus = usageReport.getReservedResources()
+ .getResourceValue(ResourceInformation.GPU_URI);
+ }
}
aggregateResourceAllocation = usageReport.getMemorySeconds()
+ " MB-seconds, " + usageReport.getVcoreSeconds()
@@ -175,6 +189,10 @@ public class AppInfo {
return allocatedMemoryMB;
}
+ public long getAllocatedGpus() {
+ return allocatedGpus;
+ }
+
public long getReservedCpuVcores() {
return reservedCpuVcores;
}
@@ -183,6 +201,10 @@ public class AppInfo {
return reservedMemoryMB;
}
+ public long getReservedGpus() {
+ return reservedGpus;
+ }
+
public float getProgress() {
return progress;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
index ba6f764..c81883d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
@@ -19,14 +19,15 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -62,35 +63,34 @@ public class MetricsOverviewTable extends HtmlBlock {
DIV<Hamlet> div = html.div().$class("metrics");
- long usedMemoryBytes = 0;
- long totalMemoryBytes = 0;
- long reservedMemoryBytes = 0;
- long usedVCores = 0;
- long totalVCores = 0;
- long reservedVCores = 0;
+ Resource usedResources;
+ Resource totalResources;
+ Resource reservedResources;
+ int allocatedContainers;
if (clusterMetrics.getCrossPartitionMetricsAvailable()) {
- ResourceInfo usedAllPartitions =
- clusterMetrics.getTotalUsedResourcesAcrossPartition();
- ResourceInfo totalAllPartitions =
- clusterMetrics.getTotalClusterResourcesAcrossPartition();
- ResourceInfo reservedAllPartitions =
- clusterMetrics.getTotalReservedResourcesAcrossPartition();
- usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB;
- totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB;
- reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB;
- usedVCores = usedAllPartitions.getvCores();
- totalVCores = totalAllPartitions.getvCores();
- reservedVCores = reservedAllPartitions.getvCores();
+ allocatedContainers =
+ clusterMetrics.getTotalAllocatedContainersAcrossPartition();
+ usedResources =
+ clusterMetrics.getTotalUsedResourcesAcrossPartition().getResource();
+ totalResources =
+ clusterMetrics.getTotalClusterResourcesAcrossPartition()
+ .getResource();
+ reservedResources =
+ clusterMetrics.getTotalReservedResourcesAcrossPartition()
+ .getResource();
// getTotalUsedResourcesAcrossPartition includes reserved resources.
- usedMemoryBytes -= reservedMemoryBytes;
- usedVCores -= reservedVCores;
+ Resources.subtractFrom(usedResources, reservedResources);
} else {
- usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB;
- totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB;
- reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB;
- usedVCores = clusterMetrics.getAllocatedVirtualCores();
- totalVCores = clusterMetrics.getTotalVirtualCores();
- reservedVCores = clusterMetrics.getReservedVirtualCores();
+ allocatedContainers = clusterMetrics.getContainersAllocated();
+ usedResources = Resource.newInstance(
+ clusterMetrics.getAllocatedMB() * BYTES_IN_MB,
+ (int) clusterMetrics.getAllocatedVirtualCores());
+ totalResources = Resource.newInstance(
+ clusterMetrics.getTotalMB() * BYTES_IN_MB,
+ (int) clusterMetrics.getTotalVirtualCores());
+ reservedResources = Resource.newInstance(
+ clusterMetrics.getReservedMB() * BYTES_IN_MB,
+ (int) clusterMetrics.getReservedVirtualCores());
}
div.h3("Cluster Metrics").
@@ -102,12 +102,9 @@ public class MetricsOverviewTable extends HtmlBlock {
th().$class("ui-state-default")._("Apps Running")._().
th().$class("ui-state-default")._("Apps Completed")._().
th().$class("ui-state-default")._("Containers Running")._().
- th().$class("ui-state-default")._("Memory Used")._().
- th().$class("ui-state-default")._("Memory Total")._().
- th().$class("ui-state-default")._("Memory Reserved")._().
- th().$class("ui-state-default")._("VCores Used")._().
- th().$class("ui-state-default")._("VCores Total")._().
- th().$class("ui-state-default")._("VCores Reserved")._().
+ th().$class("ui-state-default")._("Used Resources")._().
+ th().$class("ui-state-default")._("Total Resources")._().
+ th().$class("ui-state-default")._("Reserved Resources")._().
_().
_().
tbody().$class("ui-widget-content").
@@ -121,14 +118,10 @@ public class MetricsOverviewTable extends HtmlBlock {
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
)
).
- td(String.valueOf(
- clusterMetrics.getTotalAllocatedContainersAcrossPartition())).
- td(StringUtils.byteDesc(usedMemoryBytes)).
- td(StringUtils.byteDesc(totalMemoryBytes)).
- td(StringUtils.byteDesc(reservedMemoryBytes)).
- td(String.valueOf(usedVCores)).
- td(String.valueOf(totalVCores)).
- td(String.valueOf(reservedVCores)).
+ td(String.valueOf(allocatedContainers)).
+ td(usedResources.toString()).
+ td(totalResources.toString()).
+ td(reservedResources.toString()).
_().
_()._();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index e080f59..f9fb7dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -22,6 +22,7 @@ import com.google.inject.Inject;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -85,13 +87,17 @@ class NodesPage extends RmView {
.th(".mem", "Mem Used")
.th(".mem", "Mem Avail")
.th(".vcores", "VCores Used")
- .th(".vcores", "VCores Avail");
+ .th(".vcores", "VCores Avail")
+ .th(".gpus", "GPUs Used")
+ .th(".gpus", "GPUs Avail");
} else {
trbody.th(".containers", "Running Containers (G)")
.th(".mem", "Mem Used (G)")
.th(".mem", "Mem Avail (G)")
.th(".vcores", "VCores Used (G)")
.th(".vcores", "VCores Avail (G)")
+ .th(".gpus", "GPUs Used (G)")
+ .th(".gpus", "GPUs Avail (G)")
.th(".containers", "Running Containers (O)")
.th(".mem", "Mem Used (O)")
.th(".vcores", "VCores Used (O)")
@@ -163,6 +169,16 @@ class NodesPage extends RmView {
nodeTableData.append("\",\"<a ").append("href='" + "//" + httpAddress)
.append("'>").append(httpAddress).append("</a>\",").append("\"");
}
+ Integer gpuIndex = ResourceUtils.getResourceTypeIndex()
+ .get(ResourceInformation.GPU_URI);
+ long usedGPUs = 0;
+ long availableGPUs = 0;
+ if (gpuIndex != null) {
+ usedGPUs = info.getUsedResource().getResource()
+ .getResourceValue(ResourceInformation.GPU_URI);
+ availableGPUs = info.getAvailableResource().getResource()
+ .getResourceValue(ResourceInformation.GPU_URI);
+ }
nodeTableData.append("<br title='")
.append(String.valueOf(info.getLastHealthUpdate())).append("'>")
.append(Times.format(info.getLastHealthUpdate())).append("\",\"")
@@ -176,6 +192,10 @@ class NodesPage extends RmView {
.append("\",\"").append(String.valueOf(info.getUsedVirtualCores()))
.append("\",\"")
.append(String.valueOf(info.getAvailableVirtualCores()))
+ .append("\",\"")
+ .append(String.valueOf(usedGPUs))
+ .append("\",\"")
+ .append(String.valueOf(availableGPUs))
.append("\",\"");
// If opportunistic containers are enabled, add extra fields.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index e0baf10..31e0b72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -69,8 +69,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".runningcontainer", "Running Containers")
.th(".allocatedCpu", "Allocated CPU VCores")
.th(".allocatedMemory", "Allocated Memory MB")
+ .th(".allocatedGpu", "Allocated GPUs")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".reservedGpu", "Reserved GPUs")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -104,6 +106,7 @@ public class RMAppsBlock extends AppsBlock {
String blacklistedNodesCount = "N/A";
RMApp rmApp = rm.getRMContext().getRMApps()
.get(appAttemptId.getApplicationId());
+ boolean isAppInCompletedState = false;
if (rmApp != null) {
RMAppAttempt appAttempt = rmApp.getRMAppAttempt(appAttemptId);
Set<String> nodes =
@@ -111,6 +114,7 @@ public class RMAppsBlock extends AppsBlock {
if (nodes != null) {
blacklistedNodesCount = String.valueOf(nodes.size());
}
+ isAppInCompletedState = rmApp.isAppInCompletedStates();
}
String percent = StringUtils.format("%.1f", app.getProgress());
appsTableData
@@ -152,12 +156,18 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getAllocatedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getAllocatedMemoryMB()))
.append("\",\"")
+ .append((isAppInCompletedState && app.getAllocatedGpus() <= 0)
+ ? UNAVAILABLE : String.valueOf(app.getAllocatedGpus()))
+ .append("\",\"")
.append(app.getReservedCpuVcores() == -1 ? "N/A" : String
.valueOf(app.getReservedCpuVcores()))
.append("\",\"")
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append((isAppInCompletedState && app.getReservedGpus() <= 0)
+ ? UNAVAILABLE : String.valueOf(app.getReservedGpus()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index cc97674..3a1ec1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
- final int numberOfThInMetricsTable = 23;
- final int numberOfActualTableHeaders = 13;
+ final int numberOfThInMetricsTable = 20;
+ final int numberOfActualTableHeaders = 15;
private final int numberOfThForOpportunisticContainers = 4;
private Injector injector;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org