You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/07/01 12:11:25 UTC
[hadoop] branch trunk updated: YARN-10330. Add missing test
scenarios to TestUserGroupMappingPlacementRule and
TestAppNameMappingPlacementRule. Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 04abd0e YARN-10330. Add missing test scenarios to TestUserGroupMappingPlacementRule and TestAppNameMappingPlacementRule. Contributed by Peter Bacsko
04abd0e is described below
commit 04abd0eb17b58e321893e8651ec596e9f7ac786f
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Jul 1 14:10:55 2020 +0200
YARN-10330. Add missing test scenarios to TestUserGroupMappingPlacementRule and TestAppNameMappingPlacementRule. Contributed by Peter Bacsko
---
.../placement/TestAppNameMappingPlacementRule.java | 106 ++++++++++---
.../TestUserGroupMappingPlacementRule.java | 169 ++++++++++++++++++++-
.../scheduler/fair/SimpleGroupsMapping.java | 11 +-
3 files changed, 263 insertions(+), 23 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
index 1204213..29141ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
@@ -37,7 +37,15 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestAppNameMappingPlacementRule {
+ private static final String ROOT_QUEUE = "root";
+ private static final String Q2_QUEUE = "q2";
+ private static final String Q1_QUEUE = "q1";
+ private static final String USER_NAME = "user";
+ private static final String DEFAULT_QUEUE = "default";
+ private static final String APPLICATION_PLACEHOLDER = "%application";
+ private static final String AMBIGUOUS_QUEUE = "ambiguousQueue";
private static final String APP_NAME = "DistributedShell";
+ private static final String MAPREDUCE_APP_NAME = "MAPREDUCE";
private YarnConfiguration conf = new YarnConfiguration();
@@ -62,16 +70,20 @@ public class TestAppNameMappingPlacementRule {
CapacitySchedulerQueueManager qm =
mock(CapacitySchedulerQueueManager.class);
when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false);
+ when(qm.isAmbiguous(AMBIGUOUS_QUEUE)).thenReturn(true);
+
rule.queueManager = qm;
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
- if (inputQueue.equals("%application")) {
+ if (inputQueue.equals(APPLICATION_PLACEHOLDER)) {
inputQueue = APP_NAME;
}
asc.setQueue(inputQueue);
String appName = queueMapping.getSource();
- if (appName.equals("%application")) {
+ // to create a scenario when source != appName
+ if (appName.equals(APPLICATION_PLACEHOLDER)
+ || appName.equals(MAPREDUCE_APP_NAME)) {
appName = APP_NAME;
}
asc.setApplicationName(appName);
@@ -81,31 +93,85 @@ public class TestAppNameMappingPlacementRule {
ctx != null ? ctx.getQueue() : inputQueue);
}
- public QueueMapping queueMappingBuilder(String source, String queue) {
+ public QueueMapping getQueueMapping(String source, String queue) {
+ return getQueueMapping(source, null, queue);
+ }
+
+ public QueueMapping getQueueMapping(String source, String parent,
+ String queue) {
return QueueMapping.QueueMappingBuilder.create()
.type(QueueMapping.MappingType.APPLICATION)
.source(source)
.queue(queue)
+ .parentQueue(parent)
.build();
}
@Test
- public void testMapping() throws YarnException {
- // simple base case for mapping user to queue
- verifyQueueMapping(queueMappingBuilder(APP_NAME,
- "q1"), "user_1", "q1");
- verifyQueueMapping(queueMappingBuilder("%application", "q2"), "user_1",
- "q2");
- verifyQueueMapping(queueMappingBuilder("%application", "%application"),
- "user_1", APP_NAME);
-
- // specify overwritten, and see if user specified a queue, and it will be
- // overridden
- verifyQueueMapping(queueMappingBuilder(APP_NAME,
- "q1"), "1", "q2", "q1", true);
-
- // if overwritten not specified, it should be which user specified
- verifyQueueMapping(queueMappingBuilder(APP_NAME,
- "q1"), "1", "q2", "q2", false);
+ public void testSpecificAppNameMappedToDefinedQueue() throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME, Q1_QUEUE),
+ USER_NAME, Q1_QUEUE);
+ }
+
+ @Test
+ public void testPlaceholderAppSourceMappedToQueue() throws YarnException {
+ verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER, Q2_QUEUE),
+ USER_NAME, Q2_QUEUE);
+ }
+
+ @Test
+ public void testPlaceHolderAppSourceAndQueueMappedToAppNameQueue()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER,
+ APPLICATION_PLACEHOLDER), USER_NAME, APP_NAME);
+ }
+
+ @Test
+ public void testQueueInMappingOverridesSpecifiedQueue()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME,
+ Q1_QUEUE), USER_NAME, Q2_QUEUE, Q1_QUEUE, true);
+ }
+
+ @Test
+ public void testQueueInMappingDoesNotOverrideSpecifiedQueue()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME,
+ Q1_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
}
+
+ @Test
+ public void testDefaultQueueInMappingIsNotUsedWithoutOverride()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME,
+ DEFAULT_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
+ }
+
+ @Test
+ public void testDefaultQueueInMappingEqualsToInputQueue()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME,
+ DEFAULT_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
+ }
+
+ @Test
+ public void testMappingSourceDiffersFromInputQueue() throws YarnException {
+ verifyQueueMapping(getQueueMapping(MAPREDUCE_APP_NAME,
+ Q1_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
+ }
+
+ @Test(expected = YarnException.class)
+ public void testMappingContainsAmbiguousLeafQueueWithoutParent()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME, AMBIGUOUS_QUEUE),
+ USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
+ }
+
+ @Test
+ public void testMappingContainsAmbiguousLeafQueueWithParent()
+ throws YarnException {
+ verifyQueueMapping(getQueueMapping(APP_NAME, ROOT_QUEUE, AMBIGUOUS_QUEUE),
+ USER_NAME, DEFAULT_QUEUE, AMBIGUOUS_QUEUE, false);
+ }
+
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 5028ce6..98fba14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -149,6 +149,7 @@ public class TestUserGroupMappingPlacementRule {
ParentQueue root = mock(ParentQueue.class);
when(root.getQueuePath()).thenReturn(rootQueueName);
when(queueManager.getQueue(rootQueueName)).thenReturn(root);
+ when(queueManager.getQueueByFullName(rootQueueName)).thenReturn(root);
return root;
}
@@ -220,11 +221,13 @@ public class TestUserGroupMappingPlacementRule {
.withQueue("root.agroup.a")
.withQueue("root.asubgroup2")
.withQueue("root.bsubgroup2.b")
+ .withQueue("root.users.primarygrouponly")
+ .withQueue("root.admins.primarygrouponly")
.withManagedParentQueue("root.managedParent")
.build();
when(queueManager.getQueue(isNull())).thenReturn(null);
-
+ when(queueManager.isAmbiguous("primarygrouponly")).thenReturn(true);
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
@@ -376,6 +379,170 @@ public class TestUserGroupMappingPlacementRule {
}
@Test
+ public void testUserMappingToPrimaryGroupInvalidNestedPlaceholder()
+ throws YarnException {
+ // u:%user:%primary_group.%random, no matching queue
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%random")
+ .parentQueue("%primary_group")
+ .build())
+ .inputUser("a")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test
+ public void testUserMappingToSecondaryGroupInvalidNestedPlaceholder()
+ throws YarnException {
+ // u:%user:%secondary_group.%random, no matching queue
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%random")
+ .parentQueue("%secondary_group")
+ .build())
+ .inputUser("a")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test
+ public void testUserMappingDiffersFromSubmitterQueueDoesNotExist()
+ throws YarnException {
+ // u:a:%random, submitter: xyz, no matching queue
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("a")
+ .queue("%random")
+ .build())
+ .inputUser("xyz")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test
+ public void testSpecificUserMappingToPrimaryGroup() throws YarnException {
+ // u:a:%primary_group
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("a")
+ .queue("%primary_group")
+ .build())
+ .inputUser("a")
+ .expectedQueue("agroup")
+ .build());
+ }
+
+ @Test
+ public void testSpecificUserMappingToSecondaryGroup()
+ throws YarnException {
+ // u:a:%secondary_group
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("a")
+ .queue("%secondary_group")
+ .build())
+ .inputUser("a")
+ .expectedQueue("asubgroup2")
+ .build());
+ }
+
+ @Test
+ public void testSpecificUserMappingWithNoSecondaryGroup()
+ throws YarnException {
+ // u:nosecondarygroupuser:%secondary_group, no matching queue
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("nosecondarygroupuser")
+ .queue("%secondary_group")
+ .build())
+ .inputUser("nosecondarygroupuser")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test
+ public void testGenericUserMappingWithNoSecondaryGroup()
+ throws YarnException {
+ // u:%user:%user, no matching queue
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%user")
+ .parentQueue("%secondary_group")
+ .build())
+ .inputUser("nosecondarygroupuser")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test(expected = YarnException.class)
+ public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues()
+ throws YarnException {
+ // u:%user:%user, submitter nosecondarygroupuser, queue is ambiguous
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%user")
+ .parentQueue("%primary_group")
+ .build())
+ .inputUser("nosecondarygroupuser")
+ .build());
+ }
+
+ @Test(expected = YarnException.class)
+ public void testResolvedQueueIsNotManaged()
+ throws YarnException {
+ // u:%user:%primary_group.%user, "admins" group will be "root",
+ // resulting parent queue will be "root" which is not managed
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%user")
+ .parentQueue("%primary_group")
+ .build())
+ .inputUser("admins")
+ .build());
+ }
+
+ @Test(expected = YarnException.class)
+ public void testUserMappingToPrimaryGroupWithAmbiguousQueues()
+ throws YarnException {
+ // u:%user:%primary_group, submitter nosecondarygroupuser,
+ // queue is ambiguous
+ verifyQueueMapping(
+ QueueMappingTestDataBuilder.create()
+ .queueMapping(QueueMappingBuilder.create()
+ .type(MappingType.USER)
+ .source("%user")
+ .queue("%primary_group")
+ .build())
+ .inputUser("nosecondarygroupuser")
+ .expectedQueue("default")
+ .build());
+ }
+
+ @Test
public void testUserMappingToQueueNamedAsUsernameWithSecondaryGroupAsParentQueue()
throws YarnException {
verifyQueueMapping(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
index f7648c8..9c916e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
@@ -25,10 +25,17 @@ import java.util.List;
import org.apache.hadoop.security.GroupMappingServiceProvider;
public class SimpleGroupsMapping implements GroupMappingServiceProvider {
-
+
@Override
public List<String> getGroups(String user) {
- return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2");
+ if ("admins".equals(user)) {
+ return Arrays.asList("root");
+ } else if ("nosecondarygroupuser".equals(user)) {
+ return Arrays.asList("primarygrouponly");
+ } else {
+ return Arrays.asList(
+ user + "group", user + "subgroup1", user + "subgroup2");
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org