You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/10/20 04:46:11 UTC
[33/50] [abbrv] hadoop git commit: YARN-6546. SLS is slow while
loading 10k queues. (Yufei Gu via Haibo Chen)
YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033
Branch: refs/heads/YARN-1011
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Oct 17 16:04:19 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 1 +
.../sls/scheduler/CapacitySchedulerMetrics.java | 5 --
.../sls/scheduler/FairSchedulerMetrics.java | 17 +----
.../sls/scheduler/FifoSchedulerMetrics.java | 5 +-
.../sls/scheduler/SLSCapacityScheduler.java | 19 ------
.../yarn/sls/scheduler/SLSFairScheduler.java | 19 ------
.../yarn/sls/scheduler/SchedulerMetrics.java | 66 +++++++++-----------
7 files changed, 36 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
}
queueAppNumMap.put(queueName, appNum);
+ wrapper.getSchedulerMetrics().trackQueue(queueName);
}
private void runNewAM(String jobType, String user,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends SchedulerMetrics {
public CapacitySchedulerMetrics() {
super();
}
-
- @Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
@Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
+ protected void registerQueueMetrics(String queueName) {
+ super.registerQueueMetrics(queueName);
+
FairScheduler fair = (FairScheduler) scheduler;
final FSQueue queue = fair.getQueueManager().getQueue(queueName);
registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
}
-
- @Override
- public void untrackQueue(String queueName) {
- trackedQueues.remove(queueName);
-
- for (Metric metric: Metric.values()) {
- metrics.remove("variable.queue." + queueName + "." +
- metric.value + ".memory");
- metrics.remove("variable.queue." + queueName + "." +
- metric.value + ".vcores");
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
index 6ab2e1d..427b597 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
@@ -35,8 +35,9 @@ public class FifoSchedulerMetrics extends SchedulerMetrics {
}
@Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
+ protected void registerQueueMetrics(String queueName) {
+ super.registerQueueMetrics(queueName);
+
FifoScheduler fifo = (FifoScheduler) scheduler;
// for FifoScheduler, only DEFAULT_QUEUE
// here the three parameters doesn't affect results
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 108c2bc..6848b22 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -323,25 +323,6 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
queueName);
}
- private void initQueueMetrics(CSQueue queue) {
- if (queue instanceof LeafQueue) {
- schedulerMetrics.initQueueMetric(queue.getQueueName());
- return;
- }
-
- for (CSQueue child : queue.getChildQueues()) {
- initQueueMetrics(child);
- }
- }
- @Override
- public void serviceInit(Configuration configuration) throws Exception {
- super.serviceInit(configuration);
-
- if (metricsON) {
- initQueueMetrics(getRootQueue());
- }
- }
-
@Override
public void serviceStop() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 81f6648..8e49c51 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -306,25 +306,6 @@ public class SLSFairScheduler extends FairScheduler
queueName);
}
- private void initQueueMetrics(FSQueue queue) {
- if (queue instanceof FSLeafQueue) {
- schedulerMetrics.initQueueMetric(queue.getQueueName());
- return;
- }
-
- for (FSQueue child : queue.getChildQueues()) {
- initQueueMetrics(child);
- }
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- if (metricsON) {
- initQueueMetrics(getQueueManager().getRootQueue());
- }
- }
-
@Override
public void serviceStop() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index cabc94a..9c4d000 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -234,14 +234,35 @@ public abstract class SchedulerMetrics {
}
}
- public abstract void trackQueue(String queueName);
-
- public void untrackQueue(String queueName) {
- for (String m : queueTrackedMetrics) {
- metrics.remove("variable.queue." + queueName + "." + m);
+ /**
+ * Track a queue by registering its metrics.
+ *
+ * @param queue queue name
+ */
+ public void trackQueue(String queue) {
+ queueLock.lock();
+ try {
+ if (!isTracked(queue)) {
+ trackedQueues.add(queue);
+ registerQueueMetrics(queue);
+ }
+ } finally {
+ queueLock.unlock();
}
}
-
+
+ protected void registerQueueMetrics(String queueName) {
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+
+ for (QueueMetric queueMetric : QueueMetric.values()) {
+ String metricName = getQueueMetricName(queueName, queueMetric);
+ if (!counterMap.containsKey(metricName)) {
+ metrics.counter(metricName);
+ queueTrackedMetrics.add(metricName);
+ }
+ }
+ }
+
public boolean isTracked(String queueName) {
return trackedQueues.contains(queueName);
}
@@ -547,40 +568,13 @@ public abstract class SchedulerMetrics {
return "counter.queue." + queue + "." + metric.value;
}
- private void traceQueueIfNotTraced(String queue) {
- queueLock.lock();
- try {
- if (!isTracked(queue)) {
- trackQueue(queue);
- }
- } finally {
- queueLock.unlock();
- }
- }
-
- void initQueueMetric(String queueName){
- SortedMap<String, Counter> counterMap = metrics.getCounters();
-
- for (QueueMetric queueMetric : QueueMetric.values()) {
- String metricName = getQueueMetricName(queueName, queueMetric);
- if (!counterMap.containsKey(metricName)) {
- metrics.counter(metricName);
- counterMap = metrics.getCounters();
- }
- }
-
- traceQueueIfNotTraced(queueName);
- }
-
void updateQueueMetrics(Resource pendingResource, Resource allocatedResource,
String queueName) {
+ trackQueue(queueName);
+
SortedMap<String, Counter> counterMap = metrics.getCounters();
for(QueueMetric metric : QueueMetric.values()) {
String metricName = getQueueMetricName(queueName, metric);
- if (!counterMap.containsKey(metricName)) {
- metrics.counter(metricName);
- counterMap = metrics.getCounters();
- }
if (metric == QueueMetric.PENDING_MEMORY) {
counterMap.get(metricName).inc(pendingResource.getMemorySize());
@@ -592,8 +586,6 @@ public abstract class SchedulerMetrics {
counterMap.get(metricName).inc(allocatedResource.getVirtualCores());
}
}
-
- traceQueueIfNotTraced(queueName);
}
void updateQueueMetricsByRelease(Resource releaseResource, String queue) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org