You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/01/26 15:22:49 UTC
[hadoop] branch trunk updated: YARN-10596. Allow static definition
of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed
by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f1766e5 YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori
f1766e5 is described below
commit f1766e5bb4bccee9b6b88daa1483c01b50db330d
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Jan 26 16:22:35 2021 +0100
YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori
---
.../capacity/CapacitySchedulerQueueManager.java | 9 ++++-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 40 ++++++++++++++++++++++
...estRMWebServicesCapacitySchedDynamicConfig.java | 2 --
3 files changed, 48 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index c5ce700..00d1cda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -238,14 +238,21 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
boolean isReservableQueue = conf.isReservable(fullQueueName);
boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
fullQueueName);
+ // if a queue is eligible for auto queue creation v2
+ // it must be a ParentQueue (even if it is empty)
+ boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled(
+ fullQueueName);
boolean isDynamicParent = false;
+ // Auto created parent queues might not have static children, but they
+ // must be kept as a ParentQueue
CSQueue oldQueue = oldQueues.get(fullQueueName);
if (oldQueue instanceof ParentQueue) {
isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue();
}
- if (childQueueNames.size() == 0 && !isDynamicParent) {
+ if (childQueueNames.size() == 0 && !isDynamicParent &&
+ !isAutoQueueCreationV2Enabled) {
if (null == parent) {
throw new IllegalStateException(
"Queue configuration missing child queue names for " + queueName);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 98b6d3f..eada112 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -422,6 +422,46 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertTrue(user0.isDynamicQueue());
}
+ @Test
+ public void testChildlessParentQueueWhenAutoQueueCreationEnabled()
+ throws Exception {
+ startScheduler();
+ csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
+ csConf.setNonLabeledQueueWeight("root", 1f);
+ csConf.setNonLabeledQueueWeight("root.a", 1f);
+ csConf.setNonLabeledQueueWeight("root.b", 1f);
+ csConf.setQueues("root.a", new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
+ csConf.setAutoQueueCreationV2Enabled("root", true);
+ csConf.setAutoQueueCreationV2Enabled("root.a", true);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ CSQueue empty = cs.getQueue("root.empty-auto-parent");
+ Assert.assertTrue("empty-auto-parent is not a LeafQueue",
+ empty instanceof LeafQueue);
+ empty.stopQueue();
+
+ csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
+ csConf.setNonLabeledQueueWeight("root", 1f);
+ csConf.setNonLabeledQueueWeight("root.a", 1f);
+ csConf.setNonLabeledQueueWeight("root.b", 1f);
+ csConf.setQueues("root.a", new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
+ csConf.setAutoQueueCreationV2Enabled("root", true);
+ csConf.setAutoQueueCreationV2Enabled("root.a", true);
+ csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ empty = cs.getQueue("root.empty-auto-parent");
+ Assert.assertTrue("empty-auto-parent is not a ParentQueue",
+ empty instanceof ParentQueue);
+ Assert.assertEquals("empty-auto-parent has children",
+ 0, empty.getChildQueues().size());
+ Assert.assertTrue("empty-auto-parent is not eligible " +
+ "for auto queue creation",
+ ((ParentQueue)empty).isEligibleForAutoQueueCreation());
+ }
+
private LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
index 4d78d88..118a0df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
@@ -507,8 +507,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
if (enableAqc) {
conf.put("yarn.scheduler.capacity.root.auto-queue-creation-v2.enabled",
"true");
- conf.put("yarn.scheduler.capacity.root.default." +
- "auto-queue-creation-v2.enabled", "true");
}
return createConfiguration(conf);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org