You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/10/20 13:45:41 UTC
svn commit: r1186748 [3/3] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/ma...
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1186748&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Thu Oct 20 11:45:38 2011
@@ -0,0 +1,329 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.service.Service.STATE;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestApplicationACLs {
+
+ private static final String APP_OWNER = "owner";
+ private static final String FRIEND = "friend";
+ private static final String ENEMY = "enemy";
+ private static final String SUPER_USER = "superUser";
+ private static final String FRIENDLY_GROUP = "friendly-group";
+ private static final String SUPER_GROUP = "superGroup";
+
+ private static final Log LOG = LogFactory.getLog(TestApplicationACLs.class);
+
+ static MockRM resourceManager;
+ static Configuration conf = new YarnConfiguration();
+ final static YarnRPC rpc = YarnRPC.create(conf);
+ final static InetSocketAddress rmAddress = NetUtils
+ .createSocketAddr(conf.get(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ private static ClientRMProtocol rmClient;
+
+ private static RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(conf);
+
+ @BeforeClass
+ public static void setup() throws InterruptedException, IOException {
+ Store store = StoreFactory.getStore(conf);
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ AccessControlList adminACL = new AccessControlList("");
+ adminACL.addGroup(SUPER_GROUP);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
+ resourceManager = new MockRM(conf) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(getRMContext(), this.scheduler,
+ this.rmAppManager, this.applicationACLsManager);
+ };
+ };
+ new Thread() {
+ public void run() {
+ UserGroupInformation.createUserForTesting(ENEMY, new String[] {});
+ UserGroupInformation.createUserForTesting(FRIEND,
+ new String[] { FRIENDLY_GROUP });
+ UserGroupInformation.createUserForTesting(SUPER_USER,
+ new String[] { SUPER_GROUP });
+ resourceManager.start();
+ };
+ }.start();
+ int waitCount = 0;
+ while (resourceManager.getServiceState() == STATE.INITED
+ && waitCount++ < 60) {
+ LOG.info("Waiting for RM to start...");
+ Thread.sleep(1500);
+ }
+ if (resourceManager.getServiceState() != STATE.STARTED) {
+ // RM could have failed.
+ throw new IOException(
+ "ResourceManager failed to start. Final state is "
+ + resourceManager.getServiceState());
+ }
+
+ UserGroupInformation owner = UserGroupInformation
+ .createRemoteUser(APP_OWNER);
+ rmClient = owner.doAs(new PrivilegedExceptionAction<ClientRMProtocol>() {
+ @Override
+ public ClientRMProtocol run() throws Exception {
+ return (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
+ rmAddress, conf);
+ }
+ });
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if(resourceManager != null) {
+ resourceManager.stop();
+ }
+ }
+
+ @Test
+ public void testApplicationACLs() throws Exception {
+
+ verifyOwnerAccess();
+
+ verifySuperUserAccess();
+
+ verifyFriendAccess();
+
+ verifyEnemyAccess();
+ }
+
+ private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
+ AccessControlList modifyACL) throws Exception {
+ SubmitApplicationRequest submitRequest = recordFactory
+ .newRecordInstance(SubmitApplicationRequest.class);
+ ApplicationSubmissionContext context = recordFactory
+ .newRecordInstance(ApplicationSubmissionContext.class);
+
+ ApplicationId applicationId = rmClient.getNewApplication(
+ recordFactory.newRecordInstance(GetNewApplicationRequest.class))
+ .getApplicationId();
+ context.setApplicationId(applicationId);
+
+ Map<ApplicationAccessType, String> acls
+ = new HashMap<ApplicationAccessType, String>();
+ acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
+ acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());
+
+ ContainerLaunchContext amContainer = recordFactory
+ .newRecordInstance(ContainerLaunchContext.class);
+ Resource resource = BuilderUtils.newResource(1024);
+ amContainer.setResource(resource);
+ amContainer.setApplicationACLs(acls);
+ context.setAMContainerSpec(amContainer);
+ submitRequest.setApplicationSubmissionContext(context);
+ rmClient.submitApplication(submitRequest);
+ resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
+ return applicationId;
+ }
+
+ private ClientRMProtocol getRMClientForUser(String user)
+ throws IOException, InterruptedException {
+ UserGroupInformation userUGI = UserGroupInformation
+ .createRemoteUser(user);
+ ClientRMProtocol userClient = userUGI
+ .doAs(new PrivilegedExceptionAction<ClientRMProtocol>() {
+ @Override
+ public ClientRMProtocol run() throws Exception {
+ return (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
+ rmAddress, conf);
+ }
+ });
+ return userClient;
+ }
+
+ private void verifyOwnerAccess() throws Exception {
+
+ AccessControlList viewACL = new AccessControlList("");
+ viewACL.addGroup(FRIENDLY_GROUP);
+ AccessControlList modifyACL = new AccessControlList("");
+ modifyACL.addUser(FRIEND);
+ ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);
+
+ final GetApplicationReportRequest appReportRequest = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ appReportRequest.setApplicationId(applicationId);
+ final KillApplicationRequest finishAppRequest = recordFactory
+ .newRecordInstance(KillApplicationRequest.class);
+ finishAppRequest.setApplicationId(applicationId);
+
+ // View as owner
+ rmClient.getApplicationReport(appReportRequest);
+
+ // List apps as owner
+ Assert.assertEquals("App view by owner should list the apps!!", 1,
+ rmClient.getAllApplications(
+ recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
+ .getApplicationList().size());
+
+ // Kill app as owner
+ rmClient.forceKillApplication(finishAppRequest);
+ resourceManager.waitForState(applicationId, RMAppState.KILLED);
+ }
+
+ private void verifySuperUserAccess() throws Exception {
+
+ AccessControlList viewACL = new AccessControlList("");
+ viewACL.addGroup(FRIENDLY_GROUP);
+ AccessControlList modifyACL = new AccessControlList("");
+ modifyACL.addUser(FRIEND);
+ ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);
+
+ final GetApplicationReportRequest appReportRequest = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ appReportRequest.setApplicationId(applicationId);
+ final KillApplicationRequest finishAppRequest = recordFactory
+ .newRecordInstance(KillApplicationRequest.class);
+ finishAppRequest.setApplicationId(applicationId);
+
+ ClientRMProtocol superUserClient = getRMClientForUser(SUPER_USER);
+
+ // View as the superUser
+ superUserClient.getApplicationReport(appReportRequest);
+
+ // List apps as superUser
+ Assert.assertEquals("App view by super-user should list the apps!!", 2,
+ superUserClient.getAllApplications(
+ recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
+ .getApplicationList().size());
+
+ // Kill app as the superUser
+ superUserClient.forceKillApplication(finishAppRequest);
+ resourceManager.waitForState(applicationId, RMAppState.KILLED);
+ }
+
+ private void verifyFriendAccess() throws Exception {
+
+ AccessControlList viewACL = new AccessControlList("");
+ viewACL.addGroup(FRIENDLY_GROUP);
+ AccessControlList modifyACL = new AccessControlList("");
+ modifyACL.addUser(FRIEND);
+ ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);
+
+ final GetApplicationReportRequest appReportRequest = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ appReportRequest.setApplicationId(applicationId);
+ final KillApplicationRequest finishAppRequest = recordFactory
+ .newRecordInstance(KillApplicationRequest.class);
+ finishAppRequest.setApplicationId(applicationId);
+
+ ClientRMProtocol friendClient = getRMClientForUser(FRIEND);
+
+ // View as the friend
+ friendClient.getApplicationReport(appReportRequest);
+
+ // List apps as friend
+ Assert.assertEquals("App view by a friend should list the apps!!", 3,
+ friendClient.getAllApplications(
+ recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
+ .getApplicationList().size());
+
+ // Kill app as the friend
+ friendClient.forceKillApplication(finishAppRequest);
+ resourceManager.waitForState(applicationId, RMAppState.KILLED);
+ }
+
+ private void verifyEnemyAccess() throws Exception {
+
+ AccessControlList viewACL = new AccessControlList("");
+ viewACL.addGroup(FRIENDLY_GROUP);
+ AccessControlList modifyACL = new AccessControlList("");
+ modifyACL.addUser(FRIEND);
+ ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);
+
+ final GetApplicationReportRequest appReportRequest = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ appReportRequest.setApplicationId(applicationId);
+ final KillApplicationRequest finishAppRequest = recordFactory
+ .newRecordInstance(KillApplicationRequest.class);
+ finishAppRequest.setApplicationId(applicationId);
+
+ ClientRMProtocol enemyRmClient = getRMClientForUser(ENEMY);
+
+ // View as the enemy
+ try {
+ enemyRmClient.getApplicationReport(appReportRequest);
+ Assert.fail("App view by the enemy should fail!!");
+ } catch (YarnRemoteException e) {
+ LOG.info("Got exception while viewing app as the enemy", e);
+ Assert.assertEquals("User enemy cannot perform operation VIEW_APP on "
+ + applicationId, e.getMessage());
+ }
+
+ // List apps as enemy
+ Assert.assertEquals("App view by enemy should not list any apps!!", 0,
+ enemyRmClient.getAllApplications(
+ recordFactory.newRecordInstance(GetAllApplicationsRequest.class))
+ .getApplicationList().size());
+
+ // Kill app as the enemy
+ try {
+ enemyRmClient.forceKillApplication(finishAppRequest);
+ Assert.fail("App killing by the enemy should fail!!");
+ } catch (YarnRemoteException e) {
+ LOG.info("Got exception while killing app as the enemy", e);
+ Assert.assertEquals(
+ "User enemy cannot perform operation MODIFY_APP on "
+ + applicationId, e.getMessage());
+ }
+
+ rmClient.forceKillApplication(finishAppRequest);
+ }
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1186748&r1=1186747&r2=1186748&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Thu Oct 20 11:45:38 2011
@@ -28,6 +28,7 @@ import java.io.IOException;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
@@ -54,8 +56,17 @@ import com.google.inject.Module;
public class TestRMWebApp {
static final int GiB = 1024; // MiB
- @Test public void testControllerIndex() {
- Injector injector = WebAppTests.createMockInjector(this);
+ @Test
+ public void testControllerIndex() {
+ Injector injector = WebAppTests.createMockInjector(TestRMWebApp.class,
+ this, new Module() {
+
+ @Override
+ public void configure(Binder binder) {
+ binder.bind(ApplicationACLsManager.class).toInstance(
+ new ApplicationACLsManager(new Configuration()));
+ }
+ });
RmController c = injector.getInstance(RmController.class);
c.index();
assertEquals("Applications", c.get(TITLE, "unknown"));
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm?rev=1186748&r1=1186747&r2=1186748&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm Thu Oct 20 11:45:38 2011
@@ -176,6 +176,24 @@ Hadoop MapReduce Next Generation - Clust
* <<<conf/yarn-site.xml>>>
+ * Configurations for ResourceManager and NodeManager:
+
+*-------------------------+-------------------------+------------------------+
+|| Parameter || Value || Notes |
+*-------------------------+-------------------------+------------------------+
+| <<<yarn.acl.enable>>> | | |
+| | <<<true>>> / <<<false>>> | |
+| | | Enable ACLs? Defaults to <true>. |
+*-------------------------+-------------------------+------------------------+
+| <<<yarn.admin.acl>>> | | |
+| | Admin ACL | |
+| | | ACL to set admins on the cluster. |
+| | | ACLs are of for <comma-separated-users><space><comma-separated-groups>. |
+| | | Defaults to special value of <<*>> which means <anyone>. |
+| | | Special value of just <space> means no one has access. |
+*-------------------------+-------------------------+------------------------+
+
+
* Configurations for ResourceManager:
*-------------------------+-------------------------+------------------------+
@@ -206,17 +224,6 @@ Hadoop MapReduce Next Generation - Clust
| | <<<ResourceManager>>> Scheduler class. | |
| | | <<<CapacityScheduler>>> (recommended) or <<<FifoScheduler>>> |
*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.acl.enable>>> | | |
-| | <<<true>>> / <<<false>>> | |
-| | | Enable ACLs? Defaults to <true>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.admin.acl>>> | | |
-| | Admin ACL | |
-| | | ACL to set admins on the cluster. |
-| | | ACLs are of for <comma-separated-users><space><comma-separated-groups>. |
-| | | Defaults to special value of <<*>> which means <anyone>. |
-| | | Special value of just <space> means no one has access. |
-*-------------------------+-------------------------+------------------------+
| <<<yarn.nodemanager.remote-app-log-dir>>> | | |
| | </logs> | |
| | | HDFS directory where the application logs are moved on application |