You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by eb...@apache.org on 2021/03/25 22:44:16 UTC
[hadoop] branch branch-3.3 updated: YARN-10713. ClusterMetrics
should support custom resource capacity related metrics. Contributed by Qi
Zhu.
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 65bba8c YARN-10713. ClusterMetrics should support custom resource capacity related metrics. Contributed by Qi Zhu.
65bba8c is described below
commit 65bba8c3ed9356cb43e6d46cdf5a049b546c1608
Author: Eric Badger <eb...@verizonmedia.com>
AuthorDate: Thu Mar 25 22:35:19 2021 +0000
YARN-10713. ClusterMetrics should support custom resource capacity related metrics. Contributed by Qi Zhu.
(cherry picked from commit 19e418c10d8fbd994e5ca326bf894cab706b8a4f)
---
.../server/resourcemanager/ClusterMetrics.java | 50 +++++++++++++++-------
.../capacity/TestCSAllocateCustomResource.java | 6 ++-
2 files changed, 38 insertions(+), 18 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
index 7fe5cc9..95ef7a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import static org.apache.hadoop.metrics2.lib.Interns.info;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -35,6 +36,9 @@ import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetricsForCustomResources;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@InterfaceAudience.Private
@@ -58,10 +62,19 @@ public class ClusterMetrics {
@Metric("Vcore Utilization") MutableGaugeLong utilizedVirtualCores;
@Metric("Memory Capability") MutableGaugeLong capabilityMB;
@Metric("Vcore Capability") MutableGaugeLong capabilityVirtualCores;
- @Metric("GPU Capability") MutableGaugeLong capabilityGPUs;
private static final MetricsInfo RECORD_INFO = info("ClusterMetrics",
"Metrics for the Yarn Cluster");
+
+ private static final String CUSTOM_RESOURCE_CAPABILITY_METRIC_PREFIX =
+ "Capability.";
+ private static final String CUSTOM_RESOURCE_CAPABILITY_METRIC_DESC =
+ "NAME Capability";
+
+ private static CustomResourceMetrics customResourceMetrics;
+
+ private final CustomResourceMetricValue customResourceCapability =
+ new CustomResourceMetricValue();
private static volatile ClusterMetrics INSTANCE = null;
private static MetricsRegistry registry;
@@ -86,6 +99,17 @@ public class ClusterMetrics {
if (ms != null) {
ms.register("ClusterMetrics", "Metrics for the Yarn Cluster", INSTANCE);
}
+
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ customResourceMetrics =
+ new CustomResourceMetrics();
+ Map<String, Long> customResources =
+ customResourceMetrics.initAndGetCustomResources();
+ customResourceMetrics.
+ registerCustomResources(customResources,
+ registry, CUSTOM_RESOURCE_CAPABILITY_METRIC_PREFIX,
+ CUSTOM_RESOURCE_CAPABILITY_METRIC_DESC);
+ }
}
@VisibleForTesting
@@ -209,23 +233,20 @@ public class ClusterMetrics {
return capabilityVirtualCores.value();
}
- public long getCapabilityGPUs() {
- if (capabilityGPUs == null) {
- return 0;
- }
+ public Map<String, Long> getCustomResourceCapability() {
+ return customResourceCapability.getValues();
+ }
- return capabilityGPUs.value();
+ public void setCustomResourceCapability(Resource res) {
+ this.customResourceCapability.set(res);
}
public void incrCapability(Resource res) {
if (res != null) {
capabilityMB.incr(res.getMemorySize());
capabilityVirtualCores.incr(res.getVirtualCores());
- Integer gpuIndex = ResourceUtils.getResourceTypeIndex()
- .get(ResourceInformation.GPU_URI);
- if (gpuIndex != null) {
- capabilityGPUs.incr(res.
- getResourceValue(ResourceInformation.GPU_URI));
+ if (customResourceCapability != null) {
+ customResourceCapability.increase(res);
}
}
}
@@ -234,11 +255,8 @@ public class ClusterMetrics {
if (res != null) {
capabilityMB.decr(res.getMemorySize());
capabilityVirtualCores.decr(res.getVirtualCores());
- Integer gpuIndex = ResourceUtils.getResourceTypeIndex()
- .get(ResourceInformation.GPU_URI);
- if (gpuIndex != null) {
- capabilityGPUs.decr(res.
- getResourceValue(ResourceInformation.GPU_URI));
+ if (customResourceCapability != null) {
+ customResourceCapability.decrease(res);
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
index d6f1544..7b0254c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
@@ -231,7 +231,8 @@ public class TestCSAllocateCustomResource {
assertEquals("Cluster Capability Vcores incorrect",
metrics.getCapabilityVirtualCores(), 4 * 8);
assertEquals("Cluster Capability GPUs incorrect",
- metrics.getCapabilityGPUs(), 4 * 8);
+ (metrics.getCustomResourceCapability()
+ .get(GPU_URI)).longValue(), 4 * 8);
for (RMNode rmNode : rmNodes) {
nodeTracker.removeNode(rmNode.getNodeID());
@@ -243,7 +244,8 @@ public class TestCSAllocateCustomResource {
assertEquals("Cluster Capability Vcores incorrect",
metrics.getCapabilityVirtualCores(), 0);
assertEquals("Cluster Capability GPUs incorrect",
- metrics.getCapabilityGPUs(), 0);
+ (metrics.getCustomResourceCapability()
+ .get(GPU_URI)).longValue(), 0);
ClusterMetrics.destroy();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org