You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2019/08/28 21:14:02 UTC
[hadoop] branch branch-3.1 updated: YARN-9756: Create metric that
sums total memory/vcores preempted per round. Contributed by Manikandan R
(manirajv06).
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 51896ff YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06).
51896ff is described below
commit 51896ff7e62204128d4472ec16866e21b297826d
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Wed Aug 28 20:53:43 2019 +0000
YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06).
(cherry picked from commit d562050cec83a2bc2ffb6d109ed3d64b394b870d)
---
.../resourcemanager/scheduler/QueueMetrics.java | 23 ++++++++++++++++++++++
.../scheduler/capacity/CapacityScheduler.java | 1 +
.../TestCapacitySchedulerSurgicalPreemption.java | 5 +++++
3 files changed, 29 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 20a5a1f..4b70502 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
@@ -76,6 +77,10 @@ public class QueueMetrics implements MetricsSource {
@Metric("# of active applications") MutableGaugeInt activeApplications;
@Metric("App Attempt First Container Allocation Delay")
MutableRate appAttemptFirstContainerAllocationDelay;
+ @Metric("Aggregate total of preempted memory MB")
+ MutableCounterLong aggregateMemoryMBPreempted;
+ @Metric("Aggregate total of preempted vcores")
+ MutableCounterLong aggregateVcoresPreempted;
//Metrics updated only for "default" partition
@Metric("Allocated memory in MB") MutableGaugeLong allocatedMB;
@@ -552,6 +557,14 @@ public class QueueMetrics implements MetricsSource {
}
}
+ public void updatePreemptedResources(Resource res) {
+ aggregateMemoryMBPreempted.incr(res.getMemorySize());
+ aggregateVcoresPreempted.incr(res.getVirtualCores());
+ if (parent != null) {
+ parent.updatePreemptedResources(res);
+ }
+ }
+
public void reserveResource(String partition, String user, Resource res) {
if(partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
reserveResource(user, res);
@@ -732,4 +745,14 @@ public class QueueMetrics implements MetricsSource {
public long getAggregatePreemptedContainers() {
return aggregateContainersPreempted.value();
}
+
+ @VisibleForTesting
+ public long getAggregateMemoryMBPreempted() {
+ return aggregateMemoryMBPreempted.value();
+ }
+
+ @VisibleForTesting
+ public long getAggregateVcoresPreempted() {
+ return aggregateVcoresPreempted.value();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 80079c7..04df99c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1977,6 +1977,7 @@ public class CapacityScheduler extends
/ DateUtils.MILLIS_PER_SECOND;
qMetrics.updatePreemptedMemoryMBSeconds(mbSeconds);
qMetrics.updatePreemptedVcoreSeconds(vcSeconds);
+ qMetrics.updatePreemptedResources(containerResource);
}
@Lock(Lock.NoLock.class)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 800789a..ad3f23d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -185,6 +185,11 @@ public class TestCapacitySchedulerSurgicalPreemption
Assert.assertEquals("Number of preempted containers incorrectly recorded:",
4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers());
+ Assert.assertEquals("Amount of preempted memory incorrectly recorded:",
+ 4 * GB, cs.getQueue("root").getMetrics().getAggregateMemoryMBPreempted());
+ Assert.assertEquals("Number of preempted vcores incorrectly recorded:",
+ 4, cs.getQueue("root").getMetrics().getAggregateVcoresPreempted());
+
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org