You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/02 08:07:37 UTC
[hadoop] branch trunk updated: YARN-10604. Support auto queue
creation without mapping rules. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7d88953 YARN-10604. Support auto queue creation without mapping rules. Contributed by Andras Gyori
7d88953 is described below
commit 7d88953768411ef3e1187eaca039a2b0255ad091
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Feb 2 09:07:25 2021 +0100
YARN-10604. Support auto queue creation without mapping rules. Contributed by Andras Gyori
---
.../scheduler/capacity/CapacityScheduler.java | 18 +++++++++++--
.../TestCapacitySchedulerNewQueueAutoCreation.java | 30 +++++++++++++++++++++-
2 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 204fa73..1bb74a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -932,12 +932,26 @@ public class CapacityScheduler extends
boolean isRecovery) {
CSQueue queue = getQueue(queueName);
+ ApplicationPlacementContext fallbackContext = placementContext;
if (queue == null) {
- if (placementContext != null && placementContext.hasParentQueue()) {
+ // Even if placement rules are turned off, we still have the opportunity
+ // to auto create a queue.
+ if (placementContext == null) {
+ fallbackContext = CSQueueUtils.extractQueuePath(queueName);
+ }
+ if (fallbackContext.hasParentQueue()) {
try {
- return autoCreateLeafQueue(placementContext);
+ return autoCreateLeafQueue(fallbackContext);
} catch (YarnException | IOException e) {
+ // A null queue is expected if the placementContext is null. In order
+ // not to disrupt the control flow, if we fail to auto create a queue,
+ // we fall back to the original logic.
+ if (placementContext == null) {
+ LOG.error("Could not auto-create leaf queue " + queueName +
+ " due to : ", e);
+ return null;
+ }
if (isRecovery) {
if (!getConfiguration().shouldAppFailFast(getConfig())) {
LOG.error("Could not auto-create leaf queue " + queueName +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 0c5375e..48dba82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -18,14 +18,21 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -461,7 +468,28 @@ public class TestCapacitySchedulerNewQueueAutoCreation
"for auto queue creation",
((ParentQueue)empty).isEligibleForAutoQueueCreation());
}
-
+
+ @Test
+ public void testAutoQueueCreationWithDisabledMappingRules() throws Exception {
+ startScheduler();
+
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ // Set ApplicationPlacementContext to null in the submitted application
+ // in order to imitate a submission with mapping rules turned off
+ SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId,
+ "root.a.a1-auto.a2-auto", USER0, null);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(
+ appAttemptId, false);
+ cs.handle(addAppEvent);
+ cs.handle(addAttemptEvent);
+
+ CSQueue a2Auto = cs.getQueue("root.a.a1-auto.a2-auto");
+ Assert.assertNotNull(a2Auto);
+ }
+
+ @Test
public void testAutoCreateQueueUserLimitDisabled() throws Exception {
startScheduler();
createBasicQueueStructureAndValidate();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org