You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/01/13 16:23:27 UTC
[hadoop] branch trunk updated: YARN-9912. Capacity scheduler:
support u:user2:%secondary_group queue mapping. Contributed by Manikandan R
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 621c5ea YARN-9912. Capacity scheduler: support u:user2:%secondary_group queue mapping. Contributed by Manikandan R
621c5ea is described below
commit 621c5eac38697755cf29aa8618869c09092b6a80
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Jan 13 17:23:00 2020 +0100
YARN-9912. Capacity scheduler: support u:user2:%secondary_group queue mapping. Contributed by Manikandan R
---
.../placement/UserGroupMappingPlacementRule.java | 20 +++--
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 9 ++-
.../TestCapacitySchedulerQueueMappingFactory.java | 85 ++++++++++++++++++++++
3 files changed, 103 insertions(+), 11 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index de80410..5221ace 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -168,6 +168,11 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
break;
}
}
+
+ if (secondaryGroup == null && LOG.isDebugEnabled()) {
+ LOG.debug("User {} is not associated with any Secondary "
+ + "Group. Hence it may use the 'default' queue", user);
+ }
return secondaryGroup;
}
@@ -198,10 +203,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
validateQueueMapping(queueMapping);
return getPlacementContext(queueMapping, user);
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("User {} is not associated with any Secondary Group. "
- + "Hence it may use the 'default' queue", user);
- }
return null;
}
} else if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
@@ -219,10 +220,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
if (secondaryGroup != null) {
return getPlacementContext(mapping, secondaryGroup);
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("User {} is not associated with any Secondary "
- + "Group. Hence it may use the 'default' queue", user);
- }
return null;
}
} else {
@@ -232,6 +229,13 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
if (user.equals(mapping.source)) {
if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
return getPlacementContext(mapping, groups.getGroups(user).get(0));
+ } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
+ String secondaryGroup = getSecondaryGroup(user);
+ if (secondaryGroup != null) {
+ return getPlacementContext(mapping, secondaryGroup);
+ } else {
+ return null;
+ }
} else {
return getPlacementContext(mapping);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index 59fbb84..9527e80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -122,6 +122,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
public static final String B1 = B + ".b1";
public static final String B2 = B + ".b2";
public static final String B3 = B + ".b3";
+ public static final String B4 = B + ".b4subgroup1";
public static final String ASUBGROUP1_A = ASUBGROUP1 + ".e";
public static final String AGROUP_A = AGROUP + ".f";
public static final float A_CAPACITY = 20f;
@@ -135,8 +136,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
public static final float A2_CAPACITY = 70;
public static final float B1_CAPACITY = 60f;
public static final float B2_CAPACITY = 20f;
- public static final float B3_CAPACITY = 20f;
-
+ public static final float B3_CAPACITY = 10f;
+ public static final float B4_CAPACITY = 10f;
public static final int NODE_MEMORY = 16;
public static final int NODE1_VCORES = 16;
@@ -356,13 +357,15 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
conf.setCapacity(A2, A2_CAPACITY);
conf.setUserLimitFactor(A2, 100.0f);
- conf.setQueues(B, new String[] { "b1", "b2", "b3" });
+ conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4subgroup1" });
conf.setCapacity(B1, B1_CAPACITY);
conf.setUserLimitFactor(B1, 100.0f);
conf.setCapacity(B2, B2_CAPACITY);
conf.setUserLimitFactor(B2, 100.0f);
conf.setCapacity(B3, B3_CAPACITY);
conf.setUserLimitFactor(B3, 100.0f);
+ conf.setCapacity(B4, B4_CAPACITY);
+ conf.setUserLimitFactor(B4, 100.0f);
conf.setQueues(ASUBGROUP1, new String[] {"e"});
conf.setCapacity(ASUBGROUP1_A, 100f);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
index 4f63b8f..79b6f24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
@@ -236,8 +236,14 @@ public class TestCapacitySchedulerQueueMappingFactory {
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
"%user", getQueueMapping("%secondary_group", "%user"));
+ // u:b4:%secondary_group
+ UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 =
+ new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
+ "b4", "%secondary_group");
queueMappingsForUG.add(userQueueMapping1);
queueMappingsForUG.add(userQueueMapping2);
+ queueMappingsForUG.add(userQueueMapping3);
testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "f");
@@ -416,4 +422,83 @@ public class TestCapacitySchedulerQueueMappingFactory {
}
}
}
+
+ @Test
+ public void testFixedUserWithDynamicGroupQueue() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+
+ List<String> queuePlacementRules = new ArrayList<>();
+ queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
+ conf.setQueuePlacementRules(queuePlacementRules);
+
+ List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
+ conf.getQueueMappings();
+
+ // set queue mapping
+ List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
+ new ArrayList<>();
+
+ // u:user1:b1
+ UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
+ new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
+ "user1", "b1");
+
+ // u:user2:%primary_group
+ UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
+ new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
+ "user2", "%primary_group");
+
+ // u:b4:%secondary_group
+ UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 =
+ new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4",
+ "%secondary_group");
+
+ queueMappingsForUG.add(userQueueMapping1);
+ queueMappingsForUG.add(userQueueMapping2);
+ queueMappingsForUG.add(userQueueMapping3);
+ existingMappingsForUG.addAll(queueMappingsForUG);
+ conf.setQueueMappings(existingMappingsForUG);
+
+ //override with queue mappings
+ conf.setOverrideWithQueueMappings(true);
+
+ MockRM mockRM = null;
+ try {
+ mockRM = new MockRM(conf);
+ CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler();
+ cs.updatePlacementRules();
+ mockRM.start();
+ cs.start();
+
+ ApplicationSubmissionContext asc =
+ Records.newRecord(ApplicationSubmissionContext.class);
+ asc.setQueue("default");
+
+ List<PlacementRule> rules =
+ cs.getRMContext().getQueuePlacementManager().getPlacementRules();
+ UserGroupMappingPlacementRule r =
+ (UserGroupMappingPlacementRule) rules.get(0);
+
+ ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
+ assertEquals("Queue", "b1", ctx.getQueue());
+
+ ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2");
+ assertEquals("Queue", "user2group", ctx1.getQueue());
+
+ ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "b4");
+ assertEquals("Queue", "b4subgroup1", ctx2.getQueue());
+ } finally {
+ if (mockRM != null) {
+ mockRM.close();
+ }
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org