You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/15 13:21:23 UTC
[hadoop] branch trunk updated: Support max queues limit
configuration in new auto created queue,
consistent with old auto created. Contributed by Qi Zhu
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 852aac3 Support max queues limit configuration in new auto created queue, consistent with old auto created. Contributed by Qi Zhu
852aac3 is described below
commit 852aac34f2c4bea48400158098db833cca2b2611
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Feb 15 14:20:26 2021 +0100
Support max queues limit configuration in new auto created queue, consistent with old auto created. Contributed by Qi Zhu
---
.../capacity/CapacitySchedulerConfiguration.java | 34 ++++++++++++++++++++--
.../scheduler/capacity/ParentQueue.java | 10 +++++++
.../TestCapacitySchedulerNewQueueAutoCreation.java | 21 +++++++++++++
3 files changed, 63 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index abbc2d7..08b38a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2010,11 +2010,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
private static final String AUTO_QUEUE_CREATION_V2_PREFIX =
- "auto-queue-creation-v2";
+ "auto-queue-creation-v2.";
@Private
public static final String AUTO_QUEUE_CREATION_V2_ENABLED =
- AUTO_QUEUE_CREATION_V2_PREFIX + ".enabled";
+ AUTO_QUEUE_CREATION_V2_PREFIX + "enabled";
+
+ @Private
+ public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES =
+ AUTO_QUEUE_CREATION_V2_PREFIX + "max-queues";
+
+ @Private
+ public static final int
+ DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000;
@Private
public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false;
@@ -2130,6 +2138,28 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES);
}
+ /**
+ * Get the max number of queues that are allowed to be created under
+ * a parent queue which allowed auto creation v2.
+ *
+ * @param queuePath the parent queue's path
+ * @return the max number of queues allowed to be auto created,
+ * in new auto created.
+ */
+ @Private
+ public int getAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath) {
+ return getInt(getQueuePrefix(queuePath) +
+ AUTO_QUEUE_CREATION_V2_MAX_QUEUES,
+ DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES);
+ }
+
+ @VisibleForTesting
+ public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath,
+ int maxQueues) {
+ setInt(getQueuePrefix(queuePath) +
+ AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues);
+ }
+
@Private
public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY =
AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index b412e8a..f79ffcb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -543,6 +543,16 @@ public class ParentQueue extends AbstractCSQueue {
return queue;
}
+ // Check if the max queue limit is exceeded.
+ int maxQueues = csContext.getConfiguration().
+ getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath());
+ if (childQueues.size() >= maxQueues) {
+ throw new SchedulerDynamicEditException(
+ "Cannot auto create queue " + childQueuePath + ". Max Child "
+ + "Queue limit exceeded which is configured as: " + maxQueues
+ + " and number of child queues is: " + childQueues.size());
+ }
+
// First, check if we allow creation or not
boolean weightsAreUsed = false;
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index d48bdc1..4facf94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -544,6 +544,27 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertFalse(bAutoLeafQueue.hasChildQueues());
}
+ @Test
+ public void testAutoCreateQueueMaxQueuesLimit() throws Exception {
+ startScheduler();
+
+ csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit("root.e", 5);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ for (int i = 0; i < 5; ++i) {
+ createQueue("root.e.q_" + i);
+ }
+
+ // Check if max queue limit can't be exceeded
+ try {
+ createQueue("root.e.q_6");
+ Assert.fail("Can't exceed max queue limit.");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex
+ instanceof SchedulerDynamicEditException);
+ }
+ }
+
private LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org