You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/09 17:17:26 UTC
[hadoop] branch trunk updated: YARN-10619. CS Mapping Rule
%specified rule catches default submissions. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fc5f241 YARN-10619. CS Mapping Rule %specified rule catches default submissions. Contributed by Gergely Pollak
fc5f241 is described below
commit fc5f2415350ffe77bcb552dc18b794b9ef0208df
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Feb 9 18:16:42 2021 +0100
YARN-10619. CS Mapping Rule %specified rule catches default submissions. Contributed by Gergely Pollak
---
.../placement/CSMappingPlacementRule.java | 9 +++-
.../placement/TestCSMappingPlacementRule.java | 49 ++++++++++++++++++++++
2 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index 908498d..28d2cb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -217,7 +217,13 @@ public class CSMappingPlacementRule extends PlacementRule {
VariableContext vctx = new VariableContext();
vctx.put("%user", user);
- vctx.put("%specified", asc.getQueue());
+ //If the specified matches the default it means NO queue have been specified
+ //as per ClientRMService#submitApplication which sets the queue to default
+ //when no queue is provided.
+ //To place queues specifically to default, users must use root.default
+ if (!asc.getQueue().equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
+ vctx.put("%specified", asc.getQueue());
+ }
vctx.put("%application", asc.getApplicationName());
vctx.put("%default", "root.default");
try {
@@ -379,7 +385,6 @@ public class CSMappingPlacementRule extends PlacementRule {
asc.getApplicationName(), appQueue, overrideWithQueueMappings);
if (appQueue != null &&
!appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) &&
- !appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_FULL_NAME) &&
!overrideWithQueueMappings &&
!recovery) {
LOG.info("Have no jurisdiction over application submission '{}', " +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
index 703d517..69f56ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
@@ -425,6 +425,55 @@ public class TestCSMappingPlacementRule {
}
+ @Test
+ public void testSpecified() throws IOException {
+ ArrayList<MappingRule> rules = new ArrayList<>();
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.PlaceToQueueAction("%specified", true))
+ .setFallbackSkip()));
+
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.PlaceToQueueAction(
+ "root.ambiguous.group.tester", true))
+ .setFallbackSkip()));
+
+ rules.add(
+ new MappingRule(
+ MappingRuleMatchers.createAllMatcher(),
+ (new MappingRuleActions.RejectAction())
+ .setFallbackReject()));
+
+ CSMappingPlacementRule engine = setupEngine(true, rules);
+ ApplicationSubmissionContext appNoQueue = createApp("app");
+ ApplicationSubmissionContext appDefault = createApp("app", "default");
+ ApplicationSubmissionContext appRootDefault =
+ createApp("app", "root.default");
+ ApplicationSubmissionContext appBob =
+ createApp("app", "root.user.bob");
+
+ assertPlace("App with non specified queue should end up in " +
+ "'root.ambiguous.group.tester' because no queue was specified and " +
+ "this is the only rule matching the submission",
+ engine, appNoQueue, "alice", "root.ambiguous.group.tester");
+
+ assertPlace("App with specified 'default' should end up in " +
+ "'root.ambiguous.group.tester' because 'default' is the same as " +
+ "no queue being specified and this is the only rule matching the " +
+ "submission ",
+ engine, appDefault, "alice", "root.ambiguous.group.tester");
+
+ assertPlace("App with specified root.default should end up in " +
+ "'root.default' because root.default is specifically provided",
+ engine, appRootDefault, "alice", "root.default");
+
+ assertPlace("App with specified queue should end up in the specified " +
+ "queue 'root.user.bob'", engine, appBob, "alice", "root.user.bob");
+ }
+
private MappingRule createGroupMapping(String group, String queue) {
MappingRuleMatcher matcher = MappingRuleMatchers.createUserGroupMatcher(group);
MappingRuleAction action =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org