You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/01/13 16:16:19 UTC
[hadoop] branch trunk updated: YARN-9868. Validate %primary_group
queue in CS queue manager. Contributed by Manikandan R
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b7ef8a3 YARN-9868. Validate %primary_group queue in CS queue manager. Contributed by Manikandan R
b7ef8a3 is described below
commit b7ef8a333f6717735caeee09ae4ce67bff7285e3
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Jan 13 17:15:09 2020 +0100
YARN-9868. Validate %primary_group queue in CS queue manager. Contributed by Manikandan R
---
.../placement/UserGroupMappingPlacementRule.java | 22 ++++++++++++++++------
.../TestUserGroupMappingPlacementRule.java | 21 +++++++++++++++++++++
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 4 ++--
.../TestCapacitySchedulerQueueMappingFactory.java | 4 ++--
4 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 0caa602..de80410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -179,11 +179,15 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
- QueueMapping queueMapping =
- new QueueMapping(mapping.getType(), mapping.getSource(),
- user, groups.getGroups(user).get(0));
- validateQueueMapping(queueMapping);
- return getPlacementContext(queueMapping, user);
+ if (this.queueManager
+ .getQueue(groups.getGroups(user).get(0)) != null) {
+ QueueMapping queueMapping = new QueueMapping(mapping.getType(),
+ mapping.getSource(), user, groups.getGroups(user).get(0));
+ validateQueueMapping(queueMapping);
+ return getPlacementContext(queueMapping, user);
+ } else {
+ return null;
+ }
} else if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
@@ -203,7 +207,13 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
} else if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
- return getPlacementContext(mapping, groups.getGroups(user).get(0));
+ if (this.queueManager
+ .getQueue(groups.getGroups(user).get(0)) != null) {
+ return getPlacementContext(mapping,
+ groups.getGroups(user).get(0));
+ } else {
+ return null;
+ }
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 7590756..5cd6ea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -26,6 +27,7 @@ import java.util.Arrays;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.NullGroupsMapping;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -90,6 +92,7 @@ public class TestUserGroupMappingPlacementRule {
when(queueManager.getQueue("agroup")).thenReturn(agroup);
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
+
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
@@ -119,6 +122,20 @@ public class TestUserGroupMappingPlacementRule {
}
@Test
+ public void testNullGroupMapping() throws YarnException {
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ NullGroupsMapping.class, GroupMappingServiceProvider.class);
+ try {
+ verifyQueueMapping(
+ new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
+ "default");
+ fail("No Groups for user 'a'");
+ } catch (YarnException e) {
+ // Exception is expected as there are no groups for given user
+ }
+ }
+
+ @Test
public void testMapping() throws YarnException {
verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
@@ -131,6 +148,10 @@ public class TestUserGroupMappingPlacementRule {
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a",
"agroup");
+ // Queue "bgroup" is not configured, hence "default" should be used
+ verifyQueueMapping(
+ new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b",
+ "default");
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"),
"a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index c422dfd..59fbb84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -339,8 +339,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
// Define top-level queues
// Set childQueue for root
conf.setQueues(ROOT,
- new String[] {"a", "b", "c", "d", "esubgroup1", "asubgroup2",
- "fgroup"});
+ new String[] {"a", "b", "c", "d", "esubgroup1", "esubgroup2", "fgroup",
+ "a1group", "ggroup", "g"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
index 4cec544..4f63b8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
@@ -242,8 +242,8 @@ public class TestCapacitySchedulerQueueMappingFactory {
testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "f");
try {
- testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "h");
- fail("Leaf Queue 'h' doesn't exists");
+ testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "g");
+ fail("Queue 'g' exists, but type is not Leaf Queue");
} catch (YarnException e) {
// Exception is expected as there is no such leaf queue
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org