You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/03 23:10:56 UTC
svn commit: r1564100 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/...
Author: vinodkv
Date: Mon Feb 3 22:10:56 2014
New Revision: 1564100
URL: http://svn.apache.org/r1564100
Log:
YARN-1667. Modified RM HA handling of super users (with proxying ability) to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1564100&r1=1564099&r2=1564100&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Feb 3 22:10:56 2014
@@ -114,6 +114,39 @@ Release 2.4.0 - UNRELEASED
failover by making using of a remote configuration-provider. (Xuan Gong via
vinodkv)
+ YARN-1667. Modified RM HA handling of super users (with proxying ability) to
+ be available across RM failover by making using of a remote
+ configuration-provider. (Xuan Gong via vinodkv)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ YARN-935. Correcting pom.xml to build applicationhistoryserver module
+ successfully. (Zhijie Shen via vinodkv)
+
+ YARN-962. Fixed bug in application-history proto file and renamed it be just
+ a client proto file. (Zhijie Shen via vinodkv)
+
+ YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl
+ package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv)
+
+ YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv)
+
+ YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar
+ Vavilapalli via mayank)
+
+ YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to
+ YARN-888. (Vinod Kumar Vavilapalli via zjshen)
+
+ YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli
+ via zjshen)
+
+ YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli
+ via zjshen)
+
+ YARN-1595. Made enabling history service configurable and fixed test failures on
+ branch YARN-321. (Vinod Kumar Vavilapalli via zjshen)
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1564100&r1=1564099&r2=1564100&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Mon Feb 3 22:10:56 2014
@@ -43,6 +43,9 @@ public class YarnConfiguration extends C
@Private
public static final String YARN_SITE_XML_FILE = "yarn-site.xml";
+ @Private
+ public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
+
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
static {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1564100&r1=1564099&r2=1564100&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Mon Feb 3 22:10:56 2014
@@ -363,21 +363,22 @@ public class AdminService extends Compos
@Override
public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
- throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration");
+ throws YarnException, IOException {
+ String argName = "refreshSuperUserGroupsConfiguration";
+ UserGroupInformation user = checkAcls(argName);
- // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(),
- "refreshSuperUserGroupsConfiguration",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh super-user-groups.");
throwStandbyException();
}
- ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration());
+ Configuration conf =
+ getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
- "refreshSuperUserGroupsConfiguration", "AdminService");
+ argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshSuperUserGroupsConfigurationResponse.class);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1564100&r1=1564099&r2=1564100&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Mon Feb 3 22:10:56 2014
@@ -29,10 +29,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
@@ -188,6 +190,65 @@ public class TestRMAdminService {
Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
}
+ @Test
+ public void
+ testRefreshSuperUserGroupsWithLocalConfigurationProvider() {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+
+ try {
+ rm.adminService.refreshSuperUserGroupsConfiguration(
+ RefreshSuperUserGroupsConfigurationRequest.newInstance());
+ } catch (Exception ex) {
+ fail("Using localConfigurationProvider. Should not get any exception.");
+ }
+ }
+
+ @Test
+ public void
+ testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+
+ // clean the remoteDirectory
+ cleanRemoteDirectory();
+
+ try {
+ rm.adminService.refreshSuperUserGroupsConfiguration(
+ RefreshSuperUserGroupsConfigurationRequest.newInstance());
+ fail("FileSystemBasedConfigurationProvider is used." +
+ " Should get an exception here");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().contains(
+ "Can not find Configuration: core-site.xml"));
+ }
+
+ Configuration coreConf = new Configuration(false);
+ coreConf.set("hadoop.proxyuser.test.groups", "test_groups");
+ coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts");
+ String coreConfFile = writeConfigurationXML(coreConf,
+ "core-site.xml");
+
+ // upload the file into Remote File System
+ uploadToRemoteFileSystem(new Path(coreConfFile));
+ rm.adminService.refreshSuperUserGroupsConfiguration(
+ RefreshSuperUserGroupsConfigurationRequest.newInstance());
+ Assert.assertTrue(ProxyUsers.getProxyGroups()
+ .get("hadoop.proxyuser.test.groups").size() == 1);
+ Assert.assertTrue(ProxyUsers.getProxyGroups()
+ .get("hadoop.proxyuser.test.groups").contains("test_groups"));
+
+ Assert.assertTrue(ProxyUsers.getProxyHosts()
+ .get("hadoop.proxyuser.test.hosts").size() == 1);
+ Assert.assertTrue(ProxyUsers.getProxyHosts()
+ .get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;