You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/01/07 23:41:29 UTC
[12/50] [abbrv] hadoop git commit: YARN-4522. Queue acl can be
checked at app submission. (Jian He via wangda)
YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8310b2e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8310b2e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8310b2e9
Branch: refs/heads/HDFS-1312
Commit: 8310b2e9ff3d6804bad703c4c15458b0dfeeb4af
Parents: ab725cf
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Dec 30 15:30:12 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Dec 30 15:30:12 2015 -0800
----------------------------------------------------------------------
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 ++
.../hadoop/yarn/client/ProtocolHATestBase.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 28 +++++++++++++++++---
.../scheduler/capacity/LeafQueue.java | 8 ------
.../server/resourcemanager/TestAppManager.java | 3 ++-
.../resourcemanager/TestClientRMService.java | 2 +-
.../TestRMWebServicesAppsModification.java | 10 +++++++
9 files changed, 43 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index 2272e3e..d61bf02 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -261,7 +261,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
ugi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
- public Object run() throws YarnException {
+ public Object run() throws YarnException, IOException {
rm.getClientRMService().submitApplication(subAppRequest);
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b9789b8..4c8eecb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,8 @@ Release 2.9.0 - UNRELEASED
YARN-3480. Remove attempts that are beyond max-attempt limit from state
store. (Jun Gong via jianhe)
+ YARN-4522. Queue acl can be checked at app submission. (Jian He via wangda)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 45629b2..cf7fcc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -551,7 +551,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
@Override
public SubmitApplicationResponse submitApplication(
- SubmitApplicationRequest request) throws YarnException {
+ SubmitApplicationRequest request) throws YarnException, IOException {
resetStartFailoverFlag(true);
// make sure failover has been triggered
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index b290487..4722e1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -549,7 +549,7 @@ public class ClientRMService extends AbstractService implements
@Override
public SubmitApplicationResponse submitApplication(
- SubmitApplicationRequest request) throws YarnException {
+ SubmitApplicationRequest request) throws YarnException, IOException {
ApplicationSubmissionContext submissionContext = request
.getApplicationSubmissionContext();
ApplicationId applicationId = submissionContext.getApplicationId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 711a7a7..4344914 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -78,7 +81,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private final YarnScheduler scheduler;
private final ApplicationACLsManager applicationACLsManager;
private Configuration conf;
-
+ private boolean isAclEnabled = false;
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
@@ -97,6 +100,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
+ this.isAclEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
+ YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
}
/**
@@ -276,7 +281,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
@SuppressWarnings("unchecked")
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user) throws YarnException {
+ String user) throws YarnException, AccessControlException {
ApplicationId applicationId = submissionContext.getApplicationId();
RMAppImpl application =
@@ -325,7 +330,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user, boolean isRecovery) throws YarnException {
+ String user, boolean isRecovery)
+ throws YarnException, AccessControlException {
// Do queue mapping
if (!isRecovery) {
if (rmContext.getQueuePlacementManager() != null) {
@@ -346,6 +352,22 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
submissionContext.getQueue(), applicationId);
submissionContext.setPriority(appPriority);
+ UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
+ // Since FairScheduler queue mapping is done inside scheduler,
+ // if FairScheduler is used and the queue doesn't exist, we should not
+ // fail here because queue will be created inside FS. Ideally, FS queue
+ // mapping should be done outside scheduler too like CS.
+ // For now, exclude FS for the acl check.
+ if (!isRecovery && isAclEnabled && scheduler instanceof CapacityScheduler &&
+ !scheduler.checkAccess(userUgi, QueueACL.SUBMIT_APPLICATIONS,
+ submissionContext.getQueue()) &&
+ !scheduler.checkAccess(userUgi, QueueACL.ADMINISTER_QUEUE,
+ submissionContext.getQueue())) {
+ throw new AccessControlException(
+ "User " + user + " does not have permission to submit "
+ + applicationId + " to queue " + submissionContext.getQueue());
+ }
+
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
submissionContext.getApplicationName(), user,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index be1ba89..5c3f4b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -479,14 +479,6 @@ public class LeafQueue extends AbstractCSQueue {
String queue) throws AccessControlException {
// Careful! Locking order is important!
- // Check queue ACLs
- UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName);
- if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)
- && !hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) {
- throw new AccessControlException("User " + userName + " cannot submit" +
- " applications to queue " + getQueuePath());
- }
-
User user = null;
synchronized (this) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index c435692..075df47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
@@ -194,7 +195,7 @@ public class TestAppManager{
}
public void submitApplication(
ApplicationSubmissionContext submissionContext, String user)
- throws YarnException {
+ throws YarnException, IOException {
super.submitApplication(submissionContext, System.currentTimeMillis(),
user);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 54ebd25..a5b0b68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -893,7 +893,7 @@ public class TestClientRMService {
public void run() {
try {
rmService.submitApplication(submitRequest1);
- } catch (YarnException e) {}
+ } catch (YarnException | IOException e) {}
}
};
t.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8310b2e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index df3c2bc..61579f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -735,6 +735,16 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
client().addFilter(new LoggingFilter(System.out));
String lrKey = "example";
String queueName = "testqueue";
+
+ // create the queue
+ String[] queues = { "default", "testqueue" };
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setQueues("root", queues);
+ csconf.setCapacity("root.default", 50.0f);
+ csconf.setCapacity("root.testqueue", 50.0f);
+ rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
+
String appName = "test";
String appType = "test-type";
String urlPath = "apps";