You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/13 22:23:34 UTC
svn commit: r1568041 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-reso...
Author: vinodkv
Date: Thu Feb 13 21:23:33 2014
New Revision: 1568041
URL: http://svn.apache.org/r1568041
Log:
YARN-1676. Modified RM HA handling of user-to-group mappings to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1568041&r1=1568040&r2=1568041&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Feb 13 21:23:33 2014
@@ -184,6 +184,10 @@ Release 2.4.0 - UNRELEASED
YARN-1345. Remove FINAL_SAVING state from YarnApplicationAttemptState
(Zhijie Shen via jianhe)
+ YARN-1676. Modified RM HA handling of user-to-group mappings to
+ be available across RM failover by making using of a remote
+ configuration-provider. (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1568041&r1=1568040&r2=1568041&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Thu Feb 13 21:23:33 2014
@@ -381,21 +381,22 @@ public class AdminService extends Compos
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
RefreshUserToGroupsMappingsRequest request)
- throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
+ throws YarnException, IOException {
+ String argName = "refreshUserToGroupsMappings";
+ UserGroupInformation user = checkAcls(argName);
- // TODO (YARN-1459): Revisit handling user-groups on Standby RM
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(),
- "refreshUserToGroupsMapping",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh user-groups.");
throwStandbyException();
}
- Groups.getUserToGroupsMappingService().refresh();
- RMAuditLogger.logSuccess(user.getShortUserName(),
- "refreshUserToGroupsMappings", "AdminService");
+ Groups.getUserToGroupsMappingService(
+ getConfiguration(getConfig(),
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();
+
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshUserToGroupsMappingsResponse.class);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1568041&r1=1568040&r2=1568041&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Thu Feb 13 21:23:33 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
@@ -198,6 +199,9 @@ public class ResourceManager extends Com
this.conf =
this.configurationProvider.getConfiguration(this.conf,
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
+ .refresh();
}
// register the handlers for all AlwaysOn services using setupDispatcher().
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1568041&r1=1568040&r2=1568041&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Thu Feb 13 21:23:33 2014
@@ -24,10 +24,17 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
@@ -37,6 +44,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
@@ -366,6 +374,84 @@ public class TestRMAdminService {
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
}
+ @Test
+ public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ try {
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ } catch (Exception ex) {
+ fail("Using localConfigurationProvider. Should not get any exception.");
+ }
+ }
+
+ @Test
+ public void
+ testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch (Exception ex) {
+ // Expect exception here
+ }
+
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ List<String> groupWithInit =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+
+ // upload default configurations
+ uploadDefaultConfiguration();
+ Configuration conf = new Configuration();
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ MockUnixGroupsMapping.class,
+ GroupMappingServiceProvider.class);
+ uploadConfiguration(conf, "core-site.xml");
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ // Make sure RM will use the updated GroupMappingServiceProvider
+ List<String> groupBefore =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+ Assert.assertTrue(groupBefore.contains("test_group_A")
+ && groupBefore.contains("test_group_B")
+ && groupBefore.contains("test_group_C") && groupBefore.size() == 3);
+ Assert.assertTrue(groupWithInit.size() != groupBefore.size());
+ Assert.assertFalse(groupWithInit.contains("test_group_A")
+ || groupWithInit.contains("test_group_B")
+ || groupWithInit.contains("test_group_C"));
+
+ // update the groups
+ MockUnixGroupsMapping.updateGroups();
+
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ List<String> groupAfter =
+ Groups.getUserToGroupsMappingService(configuration).getGroups(user);
+
+ // should get the updated groups
+ Assert.assertTrue(groupAfter.contains("test_group_D")
+ && groupAfter.contains("test_group_E")
+ && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
+
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
@@ -418,4 +504,38 @@ public class TestRMAdminService {
.addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
}
+
+ private static class MockUnixGroupsMapping implements
+ GroupMappingServiceProvider {
+
+ @SuppressWarnings("serial")
+ private static List<String> group = new ArrayList<String>() {{
+ add("test_group_A");
+ add("test_group_B");
+ add("test_group_C");
+ }};
+
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ return group;
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ // Do nothing
+ }
+
+ public static void updateGroups() {
+ group.clear();
+ group.add("test_group_D");
+ group.add("test_group_E");
+ group.add("test_group_F");
+ }
+ }
+
}