You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2019/08/28 20:50:05 UTC
[hadoop] branch trunk updated: YARN-9756: Create metric that sums
total memory/vcores preempted per round. Contributed by Manikandan R
(manirajv06).
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6f2226a YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06).
6f2226a is described below
commit 6f2226a013daf30b00c6676a5f12160ac0b197b7
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Wed Aug 28 20:40:34 2019 +0000
YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06).
---
.../resourcemanager/scheduler/QueueMetrics.java | 31 ++++++++++++++++++++++
.../scheduler/QueueMetricsForCustomResources.java | 6 +++++
.../scheduler/capacity/LeafQueue.java | 2 ++
.../TestCapacitySchedulerSurgicalPreemption.java | 7 +++++
4 files changed, 46 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index c126338..ed5f546 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -77,6 +77,10 @@ public class QueueMetrics implements MetricsSource {
@Metric("# of active applications") MutableGaugeInt activeApplications;
@Metric("App Attempt First Container Allocation Delay")
MutableRate appAttemptFirstContainerAllocationDelay;
+ @Metric("Aggregate total of preempted memory MB")
+ MutableCounterLong aggregateMemoryMBPreempted;
+ @Metric("Aggregate total of preempted vcores")
+ MutableCounterLong aggregateVcoresPreempted;
//Metrics updated only for "default" partition
@Metric("Allocated memory in MB") MutableGaugeLong allocatedMB;
@@ -586,6 +590,23 @@ public class QueueMetrics implements MetricsSource {
}
}
+ public void updatePreemptedResources(Resource res) {
+ aggregateMemoryMBPreempted.incr(res.getMemorySize());
+ aggregateVcoresPreempted.incr(res.getVirtualCores());
+ if (parent != null) {
+ parent.updatePreemptedResources(res);
+ }
+ }
+
+ public void updatePreemptedForCustomResources(Resource res) {
+ if (queueMetricsForCustomResources != null) {
+ queueMetricsForCustomResources.increaseAggregatedPreempted(res);
+ }
+ if (parent != null) {
+ parent.updatePreemptedForCustomResources(res);
+ }
+ }
+
public void updatePreemptedSecondsForCustomResources(Resource res,
long seconds) {
if (queueMetricsForCustomResources != null) {
@@ -753,6 +774,16 @@ public class QueueMetrics implements MetricsSource {
return aggregateVcoreSecondsPreempted;
}
+ @VisibleForTesting
+ public long getAggregateMemoryMBPreempted() {
+ return aggregateMemoryMBPreempted.value();
+ }
+
+ @VisibleForTesting
+ public long getAggregateVcoresPreempted() {
+ return aggregateVcoresPreempted.value();
+ }
+
public long getAllocatedMB() {
return allocatedMB.value();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
index 3470858..ddfffc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
@@ -28,6 +28,8 @@ import java.util.Map;
public class QueueMetricsForCustomResources {
private final QueueMetricsCustomResource aggregatePreemptedSeconds =
new QueueMetricsCustomResource();
+ private final QueueMetricsCustomResource aggregatePreempted =
+ new QueueMetricsCustomResource();
private final QueueMetricsCustomResource allocated =
new QueueMetricsCustomResource();
private final QueueMetricsCustomResource available =
@@ -82,6 +84,10 @@ public class QueueMetricsForCustomResources {
aggregatePreemptedSeconds.increaseWithMultiplier(res, seconds);
}
+ public void increaseAggregatedPreempted(Resource res) {
+ aggregatePreempted.increase(res);
+ }
+
Map<String, Long> getAllocatedValues() {
return allocated.getValues();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index a178f9e..4d2b8e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -2222,7 +2222,9 @@ public class LeafQueue extends AbstractCSQueue {
/ DateUtils.MILLIS_PER_SECOND;
metrics.updatePreemptedMemoryMBSeconds(mbSeconds);
metrics.updatePreemptedVcoreSeconds(vcSeconds);
+ metrics.updatePreemptedResources(containerResource);
metrics.updatePreemptedSecondsForCustomResources(containerResource,
usedSeconds);
+ metrics.updatePreemptedForCustomResources(containerResource);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 800789a..c2f4511 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -185,6 +185,13 @@ public class TestCapacitySchedulerSurgicalPreemption
Assert.assertEquals("Number of preempted containers incorrectly recorded:",
4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers());
+ Assert.assertEquals("Amount of preempted memory incorrectly recorded:",
+ 4 * GB,
+ cs.getQueue("root").getMetrics().getAggregateMemoryMBPreempted());
+
+ Assert.assertEquals("Number of preempted vcores incorrectly recorded:", 4,
+ cs.getQueue("root").getMetrics().getAggregateVcoresPreempted());
+
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org