You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2020/01/02 08:43:07 UTC
[hadoop] branch trunk updated: Revert "YARN-10053. Use Shared Group
Mapping Service in Placement Rules."
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 21ada4d Revert "YARN-10053. Use Shared Group Mapping Service in Placement Rules."
21ada4d is described below
commit 21ada4d1b029e08c0729a5d54b4235d2e8399205
Author: Prabhu Joseph <pr...@apache.org>
AuthorDate: Thu Jan 2 14:11:34 2020 +0530
Revert "YARN-10053. Use Shared Group Mapping Service in Placement Rules."
This reverts commit 217b56ffdd5fa254f06734bc8cb6f04a02066f1a.
---
.../server/resourcemanager/placement/PrimaryGroupPlacementRule.java | 3 +--
.../placement/SecondaryGroupExistingPlacementRule.java | 3 +--
.../resourcemanager/placement/UserGroupMappingPlacementRule.java | 4 ++--
.../resourcemanager/scheduler/capacity/TestCapacityScheduler.java | 3 ---
.../capacity/TestCapacitySchedulerAutoCreatedQueueBase.java | 2 --
.../resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java | 5 -----
6 files changed, 4 insertions(+), 16 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
index 73e5cd0..235af34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
@@ -50,8 +50,7 @@ public class PrimaryGroupPlacementRule extends FSPlacementRule {
@Override
public boolean initialize(ResourceScheduler scheduler) throws IOException {
super.initialize(scheduler);
- groupProvider = Groups.
- getUserToGroupsMappingService(((FairScheduler)scheduler).getConfig());
+ groupProvider = new Groups(((FairScheduler)scheduler).getConfig());
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
index 9acdbcc..374714c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
@@ -53,8 +53,7 @@ public class SecondaryGroupExistingPlacementRule extends FSPlacementRule {
@Override
public boolean initialize(ResourceScheduler scheduler) throws IOException {
super.initialize(scheduler);
- groupProvider = Groups.
- getUserToGroupsMappingService(((FairScheduler)scheduler).getConfig());
+ groupProvider = new Groups(((FairScheduler)scheduler).getConfig());
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index d69272d..9eeb2ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -366,9 +366,9 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
// initialize groups if mappings are present
if (newMappings.size() > 0) {
+ Groups groups = new Groups(conf);
this.mappings = newMappings;
- this.groups = Groups.getUserToGroupsMappingService(
- ((CapacityScheduler)scheduler).getConf());
+ this.groups = groups;
this.overrideWithQueueMappings = overrideWithQueueMappings;
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index de772f1..f71a273 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -61,7 +61,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.TestGroupsCaching;
import org.apache.hadoop.security.UserGroupInformation;
@@ -5330,8 +5329,6 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
TestGroupsCaching.FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class);
config.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
"a1" +"=" + "agroup" + "");
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(config);
-
config.set(CapacitySchedulerConfiguration.QUEUE_MAPPING,
"g:agroup:%user");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index c422dfd..728c7fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.security.Groups;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -306,7 +305,6 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
TestGroupsCaching.FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class);
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group");
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
new UserGroupMappingPlacementRule.QueueMapping(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
index c6148cd..29c9d70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
@@ -29,7 +29,6 @@ import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
-import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -561,10 +560,8 @@ public class TestQueuePlacementPolicy {
sb.append(" </rule>");
sb.append("</queuePlacementPolicy>");
- // change the group resolution
CONF.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
PeriodGroupsMapping.class, GroupMappingServiceProvider.class);
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(CONF);
// User queue would be created under primary group queue, and the period
// in the group name should be converted into _dot_
createPolicy(sb.toString());
@@ -572,10 +569,8 @@ public class TestQueuePlacementPolicy {
context = placementManager.placeApplication(asc, "user1");
assertEquals("root.user1_dot_group.user1", context.getQueue());
- // undo the group resolution change
CONF.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(CONF);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org