You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/03 21:16:28 UTC
svn commit: r1564043 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/...
Author: vinodkv
Date: Mon Feb 3 20:16:28 2014
New Revision: 1564043
URL: http://svn.apache.org/r1564043
Log:
YARN-1668. Modified RM HA handling of admin-acls to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1564043&r1=1564042&r2=1564043&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Feb 3 20:16:28 2014
@@ -110,6 +110,10 @@ Release 2.4.0 - UNRELEASED
YARN-1639. Modified RM HA configuration handling to have a way of not
requiring separate configuration files for each RM. (Xuan Gong via vinodkv)
+ YARN-1668. Modified RM HA handling of admin-acls to be available across RM
+ failover by making using of a remote configuration-provider. (Xuan Gong via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1564043&r1=1564042&r2=1564043&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Mon Feb 3 20:16:28 2014
@@ -40,8 +40,10 @@ public class YarnConfiguration extends C
@Private
public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml";
+ @Private
+ public static final String YARN_SITE_XML_FILE = "yarn-site.xml";
+
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
- private static final String YARN_SITE_XML_FILE = "yarn-site.xml";
static {
Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1564043&r1=1564042&r2=1564043&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Mon Feb 3 20:16:28 2014
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
public class AdminService extends CompositeService implements
@@ -407,14 +408,22 @@ public class AdminService extends Compos
@Override
public RefreshAdminAclsResponse refreshAdminAcls(
- RefreshAdminAclsRequest request) throws YarnException {
- UserGroupInformation user = checkAcls("refreshAdminAcls");
+ RefreshAdminAclsRequest request) throws YarnException, IOException {
+ String argName = "refreshAdminAcls";
+ UserGroupInformation user = checkAcls(argName);
- Configuration conf = new Configuration();
+ if (!isRMActive()) {
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
+ adminAcl.toString(), "AdminService",
+ "ResourceManager is not active. Can not refresh user-groups.");
+ throwStandbyException();
+ }
+ Configuration conf =
+ getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE);
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
- RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class);
@@ -504,4 +513,9 @@ public class AdminService extends Compos
throws YarnException, IOException {
return this.configurationProvider.getConfiguration(confFileName);
}
+
+ @VisibleForTesting
+ public AccessControlList getAccessControlList() {
+ return this.adminAcl;
+ }
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1564043&r1=1564042&r2=1564043&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Mon Feb 3 20:16:28 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -50,6 +51,7 @@ public class TestRMAdminService {
@Before
public void setup() throws IOException {
+ Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE);
fs = FileSystem.get(configuration);
workingPath =
new Path(new File("target", this.getClass().getSimpleName()
@@ -72,6 +74,7 @@ public class TestRMAdminService {
fs.delete(workingPath, true);
fs.delete(tmpDir, true);
}
+
@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider()
throws IOException, YarnException {
@@ -95,7 +98,6 @@ public class TestRMAdminService {
@Test
public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider()
throws IOException, YarnException {
- Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE);
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
rm = new MockRM(configuration);
@@ -134,6 +136,58 @@ public class TestRMAdminService {
Assert.assertTrue(maxAppsAfter != maxAppsBefore);
}
+ @Test
+ public void testAdminAclsWithLocalConfigurationProvider() {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+
+ try {
+ rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance());
+ } catch (Exception ex) {
+ fail("Using localConfigurationProvider. Should not get any exception.");
+ }
+ }
+
+ @Test
+ public void testAdminAclsWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+
+ // clean the remoteDirectory
+ cleanRemoteDirectory();
+
+ try {
+ rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance());
+ fail("FileSystemBasedConfigurationProvider is used." +
+ " Should get an exception here");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().contains(
+ "Can not find Configuration: yarn-site.xml"));
+ }
+
+ String aclStringBefore =
+ rm.adminService.getAccessControlList().getAclString().trim();
+
+ YarnConfiguration yarnConf = new YarnConfiguration();
+ yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
+ String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml");
+
+ // upload the file into Remote File System
+ uploadToRemoteFileSystem(new Path(yarnConfFile));
+ rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance());
+
+ String aclStringAfter =
+ rm.adminService.getAccessControlList().getAclString().trim();
+
+ Assert.assertTrue(!aclStringAfter.equals(aclStringBefore));
+ Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;