You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/01/20 14:24:22 UTC
[hadoop] branch trunk updated: YARN-10578. Fix Auto Queue Creation
parent handling. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new cfe6e1f YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori
cfe6e1f is described below
commit cfe6e1f7da95d2ae05e55e5244925d296df4bbd4
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Jan 20 15:22:44 2021 +0100
YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java | 19 ----------------
.../scheduler/capacity/CapacityScheduler.java | 25 +++++++++++-----------
.../CapacitySchedulerAutoQueueHandler.java | 5 +----
.../TestCapacitySchedulerNewQueueAutoCreation.java | 3 +--
4 files changed, 14 insertions(+), 38 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index c02e1d3..bc3ff22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -154,11 +154,6 @@ public abstract class AbstractCSQueue implements CSQueue {
// is it a dynamic queue?
private boolean dynamicQueue = false;
- // When this queue has application submit to?
- // This property only applies to dynamic queue,
- // and will be used to check when the queue need to be removed.
- private long lastSubmittedTimestamp;
-
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
this(cs, cs.getConfiguration(), queueName, parent, old);
@@ -1633,18 +1628,4 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
-
- public long getLastSubmittedTimestamp() {
- return lastSubmittedTimestamp;
- }
-
- // "Tab" the queue, so this queue won't be removed because of idle timeout.
- public void signalToSubmitToQueue() {
- writeLock.lock();
- try {
- this.lastSubmittedTimestamp = System.currentTimeMillis();
- } finally {
- writeLock.unlock();
- }
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 5ee557a..204fa73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -343,7 +343,7 @@ public class CapacityScheduler extends
this.queueManager.setCapacitySchedulerContext(this);
this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler(
- this.queueManager, this.conf);
+ this.queueManager);
this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
@@ -3380,26 +3380,25 @@ public class CapacityScheduler extends
if (!StringUtils.isEmpty(parentQueueName)) {
CSQueue parentQueue = getQueue(parentQueueName);
- if (parentQueue == null) {
- throw new SchedulerDynamicEditException(
- "Could not auto-create leaf queue for " + leafQueueName
- + ". Queue mapping specifies an invalid parent queue "
- + "which does not exist " + parentQueueName);
- }
-
- if (conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) {
+ if (parentQueue != null &&
+ conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) {
// Case 1: Handle ManagedParentQueue
- AutoCreatedLeafQueue autoCreatedLeafQueue = null;
ManagedParentQueue autoCreateEnabledParentQueue =
(ManagedParentQueue) parentQueue;
- autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName,
- autoCreateEnabledParentQueue);
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ new AutoCreatedLeafQueue(
+ this, leafQueueName, autoCreateEnabledParentQueue);
addQueue(autoCreatedLeafQueue);
return autoCreatedLeafQueue;
} else {
- return autoQueueHandler.autoCreateQueue(placementContext);
+ try {
+ writeLock.lock();
+ return autoQueueHandler.autoCreateQueue(placementContext);
+ } finally {
+ writeLock.unlock();
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
index 1730021..55ab1ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
@@ -31,14 +31,11 @@ import java.util.List;
*/
public class CapacitySchedulerAutoQueueHandler {
private final CapacitySchedulerQueueManager queueManager;
- private final CapacitySchedulerConfiguration conf;
private static final int MAXIMUM_DEPTH_ALLOWED = 2;
public CapacitySchedulerAutoQueueHandler(
- CapacitySchedulerQueueManager queueManager,
- CapacitySchedulerConfiguration conf) {
+ CapacitySchedulerQueueManager queueManager) {
this.queueManager = queueManager;
- this.conf = conf;
}
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 25b2f4d..98b6d3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -83,7 +83,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
mockRM.start();
cs.start();
autoQueueHandler = new CapacitySchedulerAutoQueueHandler(
- cs.getCapacitySchedulerQueueManager(), csConf);
+ cs.getCapacitySchedulerQueueManager());
mockRM.registerNode("h1:1234", MAX_MEMORY * GB); // label = x
}
@@ -409,7 +409,6 @@ public class TestCapacitySchedulerNewQueueAutoCreation
@Test
public void testAutoQueueCreationOnAppSubmission() throws Exception {
startScheduler();
- createBasicQueueStructureAndValidate();
submitApp(cs, USER0, USER0, "root.e-auto");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org