You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/10/15 15:23:31 UTC
[hadoop] branch trunk updated: YARN-9840. Capacity scheduler: add
support for Secondary Group rule mapping. Contributed by Manikandan R
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a8ef03e YARN-9840. Capacity scheduler: add support for Secondary Group rule mapping. Contributed by Manikandan R
a8ef03e is described below
commit a8ef03e96183fad0ecc53b89f1f58aaa83a92ad8
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Oct 15 17:22:52 2019 +0200
YARN-9840. Capacity scheduler: add support for Secondary Group rule mapping. Contributed by Manikandan R
---
.../placement/QueuePlacementRuleUtils.java | 5 ++-
.../placement/UserGroupMappingPlacementRule.java | 43 +++++++++++++++++---
.../TestUserGroupMappingPlacementRule.java | 27 ++++++++++++-
.../scheduler/fair/PrimaryGroupMapping.java | 47 ++++++++++++++++++++++
.../src/site/markdown/CapacityScheduler.md | 4 +-
5 files changed, 116 insertions(+), 10 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
index adee5d7..6f2ee33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
@@ -36,6 +36,8 @@ public final class QueuePlacementRuleUtils {
public static final String PRIMARY_GROUP_MAPPING = "%primary_group";
+ public static final String SECONDARY_GROUP_MAPPING = "%secondary_group";
+
private QueuePlacementRuleUtils() {
}
@@ -100,7 +102,8 @@ public final class QueuePlacementRuleUtils {
public static boolean isStaticQueueMapping(QueueMappingEntity mapping) {
return !mapping.getQueue().contains(CURRENT_USER_MAPPING) && !mapping
- .getQueue().contains(PRIMARY_GROUP_MAPPING);
+ .getQueue().contains(PRIMARY_GROUP_MAPPING)
+ && !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING);
}
public static QueuePath extractQueuePath(String queueName)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 43a3043..71541e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -54,9 +54,12 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
public static final String PRIMARY_GROUP_MAPPING = "%primary_group";
+ public static final String SECONDARY_GROUP_MAPPING = "%secondary_group";
+
private boolean overrideWithQueueMappings = false;
private List<QueueMapping> mappings = null;
private Groups groups;
+ private CapacitySchedulerQueueManager queueManager;
@Private
public static class QueueMapping {
@@ -163,6 +166,27 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
return getPlacementContext(mapping, groups.getGroups(user).get(0));
+ } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
+ List<String> groupsList = groups.getGroups(user);
+ String secondaryGroup = null;
+ // Traverse all secondary groups (as there could be more than one
+ // and position is not guaranteed) and ensure there is queue with
+ // the same name
+ for (int i = 1; i < groupsList.size(); i++) {
+ if (this.queueManager.getQueue(groupsList.get(i)) != null) {
+ secondaryGroup = groupsList.get(i);
+ break;
+ }
+ }
+ if (secondaryGroup != null) {
+ return getPlacementContext(mapping, secondaryGroup);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User {} is not associated with any Secondary "
+ + "Group. Hence it may use the 'default' queue", user);
+ }
+ return null;
+ }
} else {
return getPlacementContext(mapping);
}
@@ -251,8 +275,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
// Get new user/group mappings
List<QueueMapping> newMappings = new ArrayList<>();
- CapacitySchedulerQueueManager queueManager =
- schedulerContext.getCapacitySchedulerQueueManager();
+ queueManager = schedulerContext.getCapacitySchedulerQueueManager();
// check if mappings refer to valid queues
for (QueueMapping mapping : queueMappings) {
@@ -365,10 +388,12 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
}
private static boolean isStaticQueueMapping(QueueMapping mapping) {
- return !mapping.getQueue().contains(
- UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mapping
- .getQueue().contains(
- UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING);
+ return !mapping.getQueue()
+ .contains(UserGroupMappingPlacementRule.CURRENT_USER_MAPPING)
+ && !mapping.getQueue()
+ .contains(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)
+ && !mapping.getQueue()
+ .contains(UserGroupMappingPlacementRule.SECONDARY_GROUP_MAPPING);
}
private static class QueuePath {
@@ -443,4 +468,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
public List<QueueMapping> getQueueMappings() {
return mappings;
}
+
+ @VisibleForTesting
+ @Private
+ public void setQueueManager(CapacitySchedulerQueueManager queueManager) {
+ this.queueManager = queueManager;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index a6143ba..589e180 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.util.Arrays;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -28,6 +31,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PrimaryGroupMapping;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
@@ -54,6 +60,10 @@ public class TestUserGroupMappingPlacementRule {
Groups groups = new Groups(conf);
UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
overwrite, Arrays.asList(queueMapping), groups);
+ CapacitySchedulerQueueManager queueManager =
+ mock(CapacitySchedulerQueueManager.class);
+ when(queueManager.getQueue("asubgroup2")).thenReturn(mock(CSQueue.class));
+ rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
asc.setQueue(inputQueue);
@@ -63,8 +73,23 @@ public class TestUserGroupMappingPlacementRule {
}
@Test
+ public void testSecondaryGroupMapping() throws YarnException {
+ verifyQueueMapping(
+ new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
+ "asubgroup2");
+
+ // PrimaryGroupMapping.class returns only primary group, no secondary groups
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ PrimaryGroupMapping.class, GroupMappingServiceProvider.class);
+
+ verifyQueueMapping(
+ new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
+ "default");
+ }
+
+ @Test
public void testMapping() throws YarnException {
- // simple base case for mapping user to queue
+
verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"),
"a", "q1");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java
new file mode 100644
index 0000000..11415b0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Group Mapping class used for test cases. Returns only primary group of the
+ * given user
+ */
+public class PrimaryGroupMapping implements GroupMappingServiceProvider {
+
+ @Override
+ public List<String> getGroups(String user) {
+ return Arrays.asList(user + "group");
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 33d2b13..511a485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -161,7 +161,7 @@ Configuration
| Property | Description |
|:---- |:---- |
-| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates [...]
+| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates [...]
| `yarn.scheduler.queue-placement-rules.app-name` | This configuration specifies the mapping of application_name to a specific queue. You can map a single application or a list of applications to queues. Syntax: `[app_name]:[queue_name][,next_mapping]*`. Here, *app_name* indicates the application name you want to do the mapping. *queue_name* indicates the queue name for which the application has to be mapped. To specify the current application's name as the app_name, %application can be used.|
| `yarn.scheduler.capacity.queue-mappings-override.enable` | This function is used to specify whether the user specified queues can be overridden. This is a Boolean value and the default value is *false*. |
@@ -170,7 +170,7 @@ Example:
```
<property>
<name>yarn.scheduler.capacity.queue-mappings</name>
- <value>u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group</value>
+ <value>u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group,u:user3:%secondary_group</value>
<description>
Here, <user1> is mapped to <queue1>, <group1> is mapped to <queue2>,
maps users to queues with the same name as user, <user2> is mapped
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org