You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/02/26 20:58:39 UTC
[hadoop] branch trunk updated: YARN-10627. Extend logging to give
more information about weight mode. Contributed by Benjamin Teke.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 947b504 YARN-10627. Extend logging to give more information about weight mode. Contributed by Benjamin Teke.
947b504 is described below
commit 947b50489d5c8b03969bbdb68e65e7f84f074101
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Fri Feb 26 21:57:54 2021 +0100
YARN-10627. Extend logging to give more information about weight mode. Contributed by Benjamin Teke.
---
.../scheduler/capacity/AbstractCSQueue.java | 9 +++
.../scheduler/capacity/LeafQueue.java | 19 +++--
.../scheduler/capacity/ParentQueue.java | 6 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 84 +++++++++++++++++-----
4 files changed, 95 insertions(+), 23 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 12ce05f..28e2d54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1633,4 +1633,13 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
+
+ protected String getCapacityOrWeightString() {
+ if (queueCapacities.getWeight() != -1) {
+ return "weight=" + queueCapacities.getWeight() + ", " +
+ "normalizedWeight=" + queueCapacities.getNormalizedWeight();
+ } else {
+ return "capacity=" + queueCapacities.getCapacity();
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6bf8d0a..104a89c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -268,9 +268,8 @@ public class LeafQueue extends AbstractCSQueue {
usersManager.updateUserWeights();
LOG.info(
- "Initializing " + getQueuePath() + "\n" + "capacity = "
- + queueCapacities.getCapacity()
- + " [= (float) configuredCapacity / 100 ]" + "\n"
+ "Initializing " + getQueuePath() + "\n" +
+ getExtendedCapacityOrWeightString() + "\n"
+ "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
+ " [= parentAbsoluteCapacity * capacity ]" + "\n"
+ "maxCapacity = " + queueCapacities.getMaximumCapacity()
@@ -486,7 +485,7 @@ public class LeafQueue extends AbstractCSQueue {
public String toString() {
readLock.lock();
try {
- return getQueuePath() + ": " + "capacity=" + queueCapacities.getCapacity()
+ return getQueuePath() + ": " + getCapacityOrWeightString()
+ ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
+ ", " + "usedResources=" + queueUsage.getUsed() + ", "
+ "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity="
@@ -499,7 +498,19 @@ public class LeafQueue extends AbstractCSQueue {
} finally {
readLock.unlock();
}
+ }
+ protected String getExtendedCapacityOrWeightString() {
+ if (queueCapacities.getWeight() != -1) {
+ return "weight = " + queueCapacities.getWeight()
+ + " [= (float) configuredCapacity (with w suffix)] " + "\n"
+ + "normalizedWeight = " + queueCapacities.getNormalizedWeight()
+ + " [= (float) configuredCapacity / sum(configuredCapacity of " +
+ "all queues under the parent)]";
+ } else {
+ return "capacity = " + queueCapacities.getCapacity()
+ + " [= (float) configuredCapacity / 100 ]";
+ }
}
@VisibleForTesting
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index f79ffcb..ce5e490 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -173,7 +173,7 @@ public class ParentQueue extends AbstractCSQueue {
((ParentQueue) parent).getQueueOrderingPolicyConfigName());
queueOrderingPolicy.setQueues(childQueues);
- LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity()
+ LOG.info(queueName + ", " + getCapacityOrWeightString()
+ ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity()
+ ", maxCapacity=" + this.queueCapacities.getMaximumCapacity()
+ ", absoluteMaxCapacity=" + this.queueCapacities
@@ -462,8 +462,8 @@ public class ParentQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
- "numChildQueue= " + childQueues.size() + ", " +
- "capacity=" + queueCapacities.getCapacity() + ", " +
+ "numChildQueue= " + childQueues.size() + ", " +
+ getCapacityOrWeightString() + ", " +
"absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
index bdf4d8d..1742ae4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
@@ -45,11 +45,16 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Set;
public class TestCapacitySchedulerWeightMode {
- private final int GB = 1024;
+ private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String A1 = A + ".a1";
+ private static final String B1 = B + ".b1";
+ private static final String B2 = B + ".b2";
private YarnConfiguration conf;
@@ -91,14 +96,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setLabeledQueueWeight(A, "x", 100);
conf.setLabeledQueueWeight(A, "y", 50);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@@ -106,7 +109,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(B, "z", 100);
// Define 2nd-level queues
- final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@@ -116,12 +118,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
- final String B1 = B + ".b1";
conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
- final String B2 = B + ".b2";
conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@@ -155,14 +155,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setLabeledQueueWeight(A, "x", 100);
conf.setLabeledQueueWeight(A, "y", 50);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@@ -170,7 +168,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(B, "z", 100);
// Define 2nd-level queues
- final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@@ -180,12 +177,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
- final String B1 = B + ".b1";
conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
- final String B2 = B + ".b2";
conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@@ -219,14 +214,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setCapacityByLabel(A, "x", 100);
conf.setCapacityByLabel(A, "y", 50);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@@ -234,7 +227,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(B, "z", 100);
// Define 2nd-level queues
- final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@@ -244,12 +236,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
- final String B1 = B + ".b1";
conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
- final String B2 = B + ".b2";
conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@@ -297,6 +287,57 @@ public class TestCapacitySchedulerWeightMode {
getCSConfWithLabelsParentUsePctChildUseWeight(conf));
}
+ /**
+ * This checks whether the parent prints the correct log about the
+ * configured mode.
+ */
+ @Test(timeout = 300000)
+ public void testGetCapacityOrWeightStringUsingWeights() throws IOException {
+ try (MockRM rm = new MockRM(
+ getCSConfWithQueueLabelsWeightOnly(conf))) {
+ rm.start();
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
+ .getCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, true);
+
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ .getCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, true);
+
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ .getExtendedCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, true);
+ }
+ }
+
+ /**
+ * This checks whether the parent prints the correct log about the
+ * configured mode.
+ */
+ @Test(timeout = 300000)
+ public void testGetCapacityOrWeightStringParentPctLeafWeights()
+ throws IOException {
+ try (MockRM rm = new MockRM(
+ getCSConfWithLabelsParentUseWeightChildUsePct(conf))) {
+ rm.start();
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
+ .getCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, true);
+
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ .getCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, false);
+
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ .getExtendedCapacityOrWeightString();
+ validateCapacityOrWeightString(capacityOrWeightString, false);
+ }
+ }
+
private void internalTestContainerAlloationWithNodeLabel(Configuration csConf)
throws Exception {
/*
@@ -449,4 +490,15 @@ public class TestCapacitySchedulerWeightMode {
}
}
}
+
+ private void validateCapacityOrWeightString(String capacityOrWeightString,
+ boolean shouldContainWeight) {
+ Assert.assertEquals(shouldContainWeight,
+ capacityOrWeightString.contains("weight"));
+ Assert.assertEquals(shouldContainWeight,
+ capacityOrWeightString.contains("normalizedWeight"));
+ Assert.assertEquals(!shouldContainWeight,
+ capacityOrWeightString.contains("capacity"));
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org