You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/04/07 09:52:33 UTC
[hadoop] branch trunk updated: YARN-10714. Remove dangling dynamic
queues on reinitialization. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9cd69c2 YARN-10714. Remove dangling dynamic queues on reinitialization. Contributed by Andras Gyori
9cd69c2 is described below
commit 9cd69c20c4aaad51ce77933ed430b66266156ca7
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Apr 7 11:52:21 2021 +0200
YARN-10714. Remove dangling dynamic queues on reinitialization. Contributed by Andras Gyori
---
.../capacity/CapacitySchedulerQueueManager.java | 43 ++++++++++++---
.../TestCapacitySchedulerNewQueueAutoCreation.java | 61 ++++++++++++++++++++++
2 files changed, 97 insertions(+), 7 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 00d1cda..5cd1490 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -340,11 +338,14 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
}
for (CSQueue queue : existingQueues.getQueues()) {
- if (!((AbstractCSQueue) queue).isDynamicQueue() && newQueues.get(
- queue.getQueuePath()) == null && !(
- queue instanceof AutoCreatedLeafQueue && conf
- .isAutoCreateChildQueueEnabled(
- queue.getParent().getQueuePath()))) {
+ boolean isDanglingDynamicQueue = isDanglingDynamicQueue(
+ newQueues, existingQueues, queue);
+ boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue)
+ && newQueues.get(queue.getQueuePath()) == null
+ && !(queue instanceof AutoCreatedLeafQueue &&
+ conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath()));
+
+ if (isRemovable) {
existingQueues.remove(queue);
}
}
@@ -435,4 +436,32 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
getQueueStateManager() {
return this.queueStateManager;
}
+
+ private boolean isDynamicQueue(CSQueue queue) {
+ return (queue instanceof AbstractCSQueue) &&
+ ((AbstractCSQueue) queue).isDynamicQueue();
+ }
+
+ private boolean isDanglingDynamicQueue(
+ CSQueueStore newQueues, CSQueueStore existingQueues,
+ CSQueue queue) {
+ if (!isDynamicQueue(queue)) {
+ return false;
+ }
+ if (queue.getParent() == null) {
+ return true;
+ }
+ if (newQueues.get(queue.getParent().getQueuePath()) != null) {
+ return false;
+ }
+ CSQueue parent = existingQueues.get(queue.getParent().getQueuePath());
+ if (parent == null) {
+ return true;
+ }
+ // A dynamic queue is dangling, if its parent is not parsed in newQueues
+ // or if its parent is not a dynamic queue. Dynamic queues are not parsed in
+ // newQueues but they are deleted automatically, so it is safe to assume
+ // that existingQueues contain valid dynamic queues.
+ return !isDynamicQueue(parent);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index c403d23..4dae4fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -968,6 +969,66 @@ public class TestCapacitySchedulerNewQueueAutoCreation
bAutoLeafQueue.getQueueInfo().getQueueName());
}
+ @Test
+ public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception {
+ startScheduler();
+
+ // Validate static parent deletion
+ createQueue("root.a.a-auto");
+ AbstractCSQueue aAuto = (AbstractCSQueue) cs.
+ getQueue("root.a.a-auto");
+ Assert.assertTrue(aAuto.isDynamicQueue());
+
+ csConf.setState("root.a", QueueState.STOPPED);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ aAuto = (AbstractCSQueue) cs.
+ getQueue("root.a.a-auto");
+ Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState());
+ csConf.setQueues("root", new String[]{"b"});
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ CSQueue aAutoNew = cs.getQueue("root.a.a-auto");
+ Assert.assertNull(aAutoNew);
+
+ submitApp(cs, USER0, "a-auto", "root.a");
+ aAutoNew = cs.getQueue("root.a.a-auto");
+ Assert.assertNotNull(aAutoNew);
+
+ // Validate static grandparent deletion
+ csConf.setQueues("root", new String[]{"a", "b"});
+ csConf.setQueues("root.a", new String[]{"a1"});
+ csConf.setAutoQueueCreationV2Enabled("root.a.a1", true);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ createQueue("root.a.a1.a1-auto");
+ CSQueue a1Auto = cs.getQueue("root.a.a1.a1-auto");
+ Assert.assertNotNull("a1-auto should exist", a1Auto);
+
+ csConf.setQueues("root", new String[]{"b"});
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ a1Auto = cs.getQueue("root.a.a1.a1-auto");
+ Assert.assertNull("a1-auto has no parent and should not exist", a1Auto);
+
+ // Validate dynamic parent deletion
+ csConf.setState("root.b", QueueState.STOPPED);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ csConf.setAutoQueueCreationV2Enabled("root.b", true);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ createQueue("root.b.b-auto-parent.b-auto-leaf");
+ CSQueue bAutoParent = cs.getQueue("root.b.b-auto-parent");
+ Assert.assertNotNull("b-auto-parent should exist", bAutoParent);
+ ParentQueue b = (ParentQueue) cs.getQueue("root.b");
+ b.removeChildQueue(bAutoParent);
+
+ cs.reinitialize(csConf, mockRM.getRMContext());
+
+ bAutoParent = cs.getQueue("root.b.b-auto-parent");
+ Assert.assertNull("b-auto-parent should not exist ", bAutoParent);
+ CSQueue bAutoLeaf = cs.getQueue("root.b.b-auto-parent.b-auto-leaf");
+ Assert.assertNull("b-auto-leaf should not exist " +
+ "when its dynamic parent is removed", bAutoLeaf);
+ }
+
protected LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org