You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/09/05 21:02:10 UTC
[hadoop] branch trunk updated: YARN-9810. Add queue
capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0ccf4b0 YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
0ccf4b0 is described below
commit 0ccf4b0fe16a8c879a560f2a612a3185eb2df72b
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Thu Sep 5 14:01:42 2019 -0700
YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
---
.../scheduler/capacity/CSQueueMetrics.java | 39 ++++++++++++++++++++++
.../scheduler/capacity/CSQueueUtils.java | 6 ++++
.../scheduler/capacity/TestCapacityScheduler.java | 18 ++++++++++
3 files changed, 63 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 8382665..c50a1b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -55,6 +55,14 @@ public class CSQueueMetrics extends QueueMetrics {
MutableGaugeLong maxCapacityMB;
@Metric("Maximum CPU in virtual cores")
MutableGaugeInt maxCapacityVCores;
+ @Metric("Guaranteed capacity in percentage relative to parent")
+ private MutableGaugeFloat guaranteedCapacity;
+ @Metric("Guaranteed capacity in percentage relative to total partition")
+ private MutableGaugeFloat guaranteedAbsoluteCapacity;
+ @Metric("Maximum capacity in percentage relative to parent")
+ private MutableGaugeFloat maxCapacity;
+ @Metric("Maximum capacity in percentage relative to total partition")
+ private MutableGaugeFloat maxAbsoluteCapacity;
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
@@ -204,4 +212,35 @@ public class CSQueueMetrics extends QueueMetrics {
return metrics;
}
+ public float getGuaranteedCapacity() {
+ return guaranteedCapacity.value();
+ }
+
+ public float getGuaranteedAbsoluteCapacity() {
+ return guaranteedAbsoluteCapacity.value();
+ }
+
+ public void setGuaranteedCapacities(String partition, float capacity,
+ float absoluteCapacity) {
+ if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+ guaranteedCapacity.set(capacity);
+ guaranteedAbsoluteCapacity.set(absoluteCapacity);
+ }
+ }
+
+ public float getMaxCapacity() {
+ return maxCapacity.value();
+ }
+
+ public float getMaxAbsoluteCapacity() {
+ return maxAbsoluteCapacity.value();
+ }
+
+ public void setMaxCapacities(String partition, float capacity,
+ float absoluteCapacity) {
+ if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+ maxCapacity.set(capacity);
+ maxAbsoluteCapacity.set(absoluteCapacity);
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 5c5f30f..715f80b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -336,5 +336,11 @@ public class CSQueueUtils {
queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown(
partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition),
queue.getMinimumAllocation()));
+ queue.getMetrics().setGuaranteedCapacities(partition,
+ queue.getQueueCapacities().getCapacity(partition),
+ queue.getQueueCapacities().getAbsoluteCapacity(partition));
+ queue.getMetrics().setMaxCapacities(partition,
+ queue.getQueueCapacities().getMaximumCapacity(partition),
+ queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition));
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8036a21..fb323c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -195,6 +195,8 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
private ResourceManager resourceManager = null;
private RMContext mockContext;
+ private static final double DELTA = 0.000001;
+
@Before
public void setUp() throws Exception {
resourceManager = new ResourceManager() {
@@ -5502,6 +5504,22 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+ assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
+ .getMetrics()).getGuaranteedCapacity(), DELTA);
+ assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
+ .getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
+ assertEquals(B1_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("b1")
+ .getMetrics()).getGuaranteedCapacity(), DELTA);
+ assertEquals((B_CAPACITY / 100) * (B1_CAPACITY / 100), ((CSQueueMetrics)cs
+ .getQueue("b1").getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
+ assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
+ .getMaxCapacity(), DELTA);
+ assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
+ .getMaxAbsoluteCapacity(), DELTA);
+ assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
+ .getMaxCapacity(), DELTA);
+ assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
+ .getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org