You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/04/12 22:52:56 UTC
[31/50] hadoop git commit: YARN-6195. Export UsedCapacity and
AbsoluteUsedCapacity to JMX. Contributed by Benson Qiu
YARN-6195. Export UsedCapacity and AbsoluteUsedCapacity to JMX. Contributed by Benson Qiu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e065f2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e065f2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e065f2e
Branch: refs/heads/HADOOP-13345
Commit: 0e065f2edead41f6f894c9d70def389af1c0f239
Parents: 2fd568f
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Tue Apr 11 08:44:18 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Tue Apr 11 08:44:18 2017 -0500
----------------------------------------------------------------------
.../hadoop/metrics2/lib/MetricsRegistry.java | 26 +++++++
.../hadoop/metrics2/lib/MutableGaugeFloat.java | 80 ++++++++++++++++++++
.../metrics2/lib/MutableMetricsFactory.java | 3 +
.../metrics2/lib/TestMetricsAnnotations.java | 3 +
.../metrics2/lib/TestMetricsRegistry.java | 4 +-
.../hadoop/metrics2/lib/TestMutableMetrics.java | 2 +
.../scheduler/capacity/AbstractCSQueue.java | 20 ++---
.../scheduler/capacity/CSQueue.java | 14 ----
.../scheduler/capacity/CSQueueMetrics.java | 21 +++++
.../scheduler/capacity/CSQueueUtils.java | 32 +++++---
.../scheduler/capacity/LeafQueue.java | 2 +-
.../scheduler/capacity/ParentQueue.java | 2 +-
.../scheduler/capacity/ReservationQueue.java | 2 +-
13 files changed, 168 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
index 0af45a6..7070869 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
@@ -143,6 +143,7 @@ public class MetricsRegistry {
public MutableGaugeInt newGauge(String name, String desc, int iVal) {
return newGauge(Interns.info(name, desc), iVal);
}
+
/**
* Create a mutable integer gauge
* @param info metadata of the metric
@@ -181,6 +182,30 @@ public class MetricsRegistry {
}
/**
+ * Create a mutable float gauge
+ * @param name of the metric
+ * @param desc metric description
+ * @param iVal initial value
+ * @return a new gauge object
+ */
+ public MutableGaugeFloat newGauge(String name, String desc, float iVal) {
+ return newGauge(Interns.info(name, desc), iVal);
+ }
+
+ /**
+ * Create a mutable float gauge
+ * @param info metadata of the metric
+ * @param iVal initial value
+ * @return a new gauge object
+ */
+ public synchronized MutableGaugeFloat newGauge(MetricsInfo info, float iVal) {
+ checkMetricName(info.name());
+ MutableGaugeFloat ret = new MutableGaugeFloat(info, iVal);
+ metricsMap.put(info.name(), ret);
+ return ret;
+ }
+
+ /**
* Create a mutable metric that estimates quantiles of a stream of values
* @param name of the metric
* @param desc metric description
@@ -420,4 +445,5 @@ public class MetricsRegistry {
.add("info", metricsInfo).add("tags", tags()).add("metrics", metrics())
.toString();
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
new file mode 100644
index 0000000..b16eda2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.lib;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+
+/**
+ * A mutable float gauge.
+ */
+public class MutableGaugeFloat extends MutableGauge {
+
+ private AtomicInteger value = new AtomicInteger();
+
+ MutableGaugeFloat(MetricsInfo info, float initValue) {
+ super(info);
+ this.value.set(Float.floatToIntBits(initValue));
+ }
+
+ public float value() {
+ return Float.intBitsToFloat(value.get());
+ }
+
+ @Override
+ public void incr() {
+ incr(1.0f);
+ }
+
+ @Override
+ public void decr() {
+ incr(-1.0f);
+ }
+
+ @Override
+ public void snapshot(MetricsRecordBuilder builder, boolean all) {
+ if (all || changed()) {
+ builder.addGauge(info(), value());
+ clearChanged();
+ }
+ }
+
+ public void set(float value) {
+ this.value.set(Float.floatToIntBits(value));
+ setChanged();
+ }
+
+ private final boolean compareAndSet(float expect, float update) {
+ return value.compareAndSet(Float.floatToIntBits(expect),
+ Float.floatToIntBits(update));
+ }
+
+ private void incr(float delta) {
+ while (true) {
+ float current = value.get();
+ float next = current + delta;
+ if (compareAndSet(current, next)) {
+ setChanged();
+ return;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
index a576772..7900332 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
@@ -60,6 +60,9 @@ public class MutableMetricsFactory {
if (cls == MutableGaugeLong.class) {
return registry.newGauge(info, 0L);
}
+ if (cls == MutableGaugeFloat.class) {
+ return registry.newGauge(info, 0f);
+ }
if (cls == MutableRate.class) {
return registry.newRate(info.name(), info.description(),
annotation.always());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java
index 5b75e33..00c2165 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsAnnotations.java
@@ -39,6 +39,7 @@ public class TestMetricsAnnotations {
@Metric({"Counter2", "Counter2 desc"}) MutableCounterLong c2;
@Metric MutableGaugeInt g1, g2;
@Metric("g3 desc") MutableGaugeLong g3;
+ @Metric("g4 desc") MutableGaugeFloat g4;
@Metric MutableRate r1;
@Metric MutableStat s1;
@Metric MutableRates rs1;
@@ -53,6 +54,7 @@ public class TestMetricsAnnotations {
metrics.g1.incr();
metrics.g2.incr();
metrics.g3.incr();
+ metrics.g4.incr();
metrics.r1.add(1);
metrics.s1.add(1);
metrics.rs1.add("rs1", 1);
@@ -64,6 +66,7 @@ public class TestMetricsAnnotations {
verify(rb).addGauge(info("G1", "G1"), 1);
verify(rb).addGauge(info("G2", "G2"), 1);
verify(rb).addGauge(info("G3", "g3 desc"), 1L);
+ verify(rb).addGauge(info("G4", "g4 desc"), 1f);
verify(rb).addCounter(info("R1NumOps", "Number of ops for r1"), 1L);
verify(rb).addGauge(info("R1AvgTime", "Average time for r1"), 1.0);
verify(rb).addCounter(info("S1NumOps", "Number of ops for s1"), 1L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
index d916928..73ea43f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
@@ -42,13 +42,15 @@ public class TestMetricsRegistry {
r.newCounter("c2", "c2 desc", 2L);
r.newGauge("g1", "g1 desc", 3);
r.newGauge("g2", "g2 desc", 4L);
+ r.newGauge("g3", "g3 desc", 5f);
r.newStat("s1", "s1 desc", "ops", "time");
- assertEquals("num metrics in registry", 5, r.metrics().size());
+ assertEquals("num metrics in registry", 6, r.metrics().size());
assertTrue("c1 found", r.get("c1") instanceof MutableCounterInt);
assertTrue("c2 found", r.get("c2") instanceof MutableCounterLong);
assertTrue("g1 found", r.get("g1") instanceof MutableGaugeInt);
assertTrue("g2 found", r.get("g2") instanceof MutableGaugeLong);
+ assertTrue("g3 found", r.get("g3") instanceof MutableGaugeFloat);
assertTrue("s1 found", r.get("s1") instanceof MutableStat);
expectMetricsException("Metric name c1 already exists", new Runnable() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
index 1faa361..fd716ae 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
@@ -59,6 +59,7 @@ public class TestMutableMetrics {
registry.newCounter("c2", "long counter", 2L);
registry.newGauge("g1", "int gauge", 3);
registry.newGauge("g2", "long gauge", 4L);
+ registry.newGauge("g3", "float gauge", 5f);
registry.newStat("s1", "stat", "Ops", "Time", true).add(0);
registry.newRate("s2", "stat", false).add(0);
@@ -74,6 +75,7 @@ public class TestMutableMetrics {
verify(mb).addCounter(info("c2", "long counter"), 2L);
verify(mb).addGauge(info("g1", "int gauge"), 3);
verify(mb).addGauge(info("g2", "long gauge"), 4L);
+ verify(mb).addGauge(info("g3", "float gauge"), 5f);
verify(mb).addCounter(info("S1NumOps", "Number of ops for stat"), 1L);
verify(mb).addGauge(eq(info("S1AvgTime", "Average time for stat")),
eq(0.0, EPSILON));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index aa60c9c..1643390 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -228,16 +228,6 @@ public abstract class AbstractCSQueue implements CSQueue {
null, null, Server.getRemoteAddress(), null));
}
- @Override
- public void setUsedCapacity(float usedCapacity) {
- queueCapacities.setUsedCapacity(usedCapacity);
- }
-
- @Override
- public void setAbsoluteUsedCapacity(float absUsedCapacity) {
- queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity);
- }
-
/**
* Set maximum capacity - used only for testing.
* @param maximumCapacity new max capacity
@@ -309,7 +299,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Update metrics
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ this, labelManager, null);
// Check if labels of this queue is a subset of parent queue, only do this
// when we not root
@@ -461,7 +451,7 @@ public abstract class AbstractCSQueue implements CSQueue {
++numContainers;
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, nodePartition);
+ this, labelManager, nodePartition);
} finally {
writeLock.unlock();
}
@@ -474,7 +464,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodePartition, resource);
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, nodePartition);
+ this, labelManager, nodePartition);
--numContainers;
} finally {
@@ -735,7 +725,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.incUsed(nodeLabel, resourceToInc);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- minimumAllocation, queueUsage, queueCapacities, nodeLabel);
+ nodeLabel, this);
if (null != parent) {
parent.incUsedResource(nodeLabel, resourceToInc, null);
}
@@ -751,7 +741,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodeLabel, resourceToDec);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- minimumAllocation, queueUsage, queueCapacities, nodeLabel);
+ nodeLabel, this);
if (null != parent) {
parent.decUsedResource(nodeLabel, resourceToDec, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 6d30386..c6726ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -121,20 +121,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
public float getAbsoluteUsedCapacity();
/**
- * Set used capacity of the queue.
- * @param usedCapacity
- * used capacity of the queue
- */
- public void setUsedCapacity(float usedCapacity);
-
- /**
- * Set absolute used capacity of the queue.
- * @param absUsedCapacity
- * absolute used capacity of the queue
- */
- public void setAbsoluteUsedCapacity(float absUsedCapacity);
-
- /**
* Get the current used capacity of nodes without label(s) of the queue
* and it's children (if any).
* @return queue used capacity
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 58c3984..a601b7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -39,6 +40,10 @@ public class CSQueueMetrics extends QueueMetrics {
MutableGaugeLong usedAMResourceMB;
@Metric("Used AM CPU limit in virtual cores")
MutableGaugeLong usedAMResourceVCores;
+ @Metric("Percent of Capacity Used")
+ MutableGaugeFloat usedCapacity;
+ @Metric("Percent of Absolute Capacity Used")
+ MutableGaugeFloat absoluteUsedCapacity;
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
@@ -91,6 +96,22 @@ public class CSQueueMetrics extends QueueMetrics {
}
}
+ public float getUsedCapacity() {
+ return usedCapacity.value();
+ }
+
+ public void setUsedCapacity(float usedCapacity) {
+ this.usedCapacity.set(usedCapacity);
+ }
+
+ public float getAbsoluteUsedCapacity() {
+ return absoluteUsedCapacity.value();
+ }
+
+ public void setAbsoluteUsedCapacity(Float absoluteUsedCapacity) {
+ this.absoluteUsedCapacity.set(absoluteUsedCapacity);
+ }
+
public synchronized static CSQueueMetrics forQueue(String queueName,
Queue parent, boolean enableUserMetrics, Configuration conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index d5cdb32..ba22541 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -181,9 +180,12 @@ class CSQueueUtils {
* used resource for all partitions of this queue.
*/
public static void updateUsedCapacity(final ResourceCalculator rc,
- final Resource totalPartitionResource, final Resource minimumAllocation,
- ResourceUsage queueResourceUsage, QueueCapacities queueCapacities,
- String nodePartition) {
+ final Resource totalPartitionResource, String nodePartition,
+ AbstractCSQueue childQueue) {
+ QueueCapacities queueCapacities = childQueue.getQueueCapacities();
+ CSQueueMetrics queueMetrics = childQueue.getMetrics();
+ ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
+ Resource minimumAllocation = childQueue.getMinimumAllocation();
float absoluteUsedCapacity = 0.0f;
float usedCapacity = 0.0f;
float reservedCapacity = 0.0f;
@@ -225,8 +227,18 @@ class CSQueueUtils {
queueCapacities.setReservedCapacity(nodePartition, reservedCapacity);
queueCapacities
.setAbsoluteReservedCapacity(nodePartition, absoluteReservedCapacity);
+
+ // QueueMetrics does not support per-label capacities,
+ // so we report values only for the default partition.
+ if (nodePartition.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ queueMetrics.setUsedCapacity(
+ queueCapacities.getUsedCapacity(RMNodeLabelsManager.NO_LABEL));
+ queueMetrics.setAbsoluteUsedCapacity(
+ queueCapacities.getAbsoluteUsedCapacity(
+ RMNodeLabelsManager.NO_LABEL));
+ }
}
-
+
private static Resource getMaxAvailableResourceToQueue(
final ResourceCalculator rc, RMNodeLabelsManager nlm, CSQueue queue,
Resource cluster) {
@@ -270,22 +282,22 @@ class CSQueueUtils {
*/
@Lock(CSQueue.class)
public static void updateQueueStatistics(
- final ResourceCalculator rc, final Resource cluster, final Resource minimumAllocation,
- final CSQueue childQueue, final RMNodeLabelsManager nlm,
+ final ResourceCalculator rc, final Resource cluster,
+ final AbstractCSQueue childQueue, final RMNodeLabelsManager nlm,
final String nodePartition) {
QueueCapacities queueCapacities = childQueue.getQueueCapacities();
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
-
+
if (nodePartition == null) {
for (String partition : Sets.union(
queueCapacities.getNodePartitionsSet(),
queueResourceUsage.getNodePartitionsSet())) {
updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
- minimumAllocation, queueResourceUsage, queueCapacities, partition);
+ partition, childQueue);
}
} else {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
- minimumAllocation, queueResourceUsage, queueCapacities, nodePartition);
+ nodePartition, childQueue);
}
// Update queue metrics w.r.t node labels. In a generic way, we can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index fa515da..9059ef0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1676,7 +1676,7 @@ public class LeafQueue extends AbstractCSQueue {
// Update metrics
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ this, labelManager, null);
// queue metrics are updated, more resource may be available
// activate the pending applications if possible
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 1579472..91fedbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -840,7 +840,7 @@ public class ParentQueue extends AbstractCSQueue {
}
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ this, labelManager, null);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e065f2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index faeb37e..3d1b731 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -64,7 +64,7 @@ public class ReservationQueue extends LeafQueue {
}
super.reinitialize(newlyParsedQueue, clusterResource);
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ this, labelManager, null);
updateQuotas(parent.getUserLimitForReservation(),
parent.getUserLimitFactor(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org