You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ah...@apache.org on 2021/10/21 17:06:07 UTC
[hadoop] branch branch-3.3 updated: YARN-1115: Provide optional
means for a scheduler to check real user ACLs. Contributed by Eric Payne
(epayne)
This is an automated email from the ASF dual-hosted git repository.
ahussein pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 742d88b YARN-1115: Provide optional means for a scheduler to check real user ACLs. Contributed by Eric Payne (epayne)
742d88b is described below
commit 742d88b1c6948c9efa0a046cacd9718a9c526a88
Author: Ahmed Hussein <a...@ahussein.me>
AuthorDate: Thu Oct 21 17:04:29 2021 +0000
YARN-1115: Provide optional means for a scheduler to check real user ACLs. Contributed by Eric Payne (epayne)
---
.../server/resourcemanager/ClientRMService.java | 6 +-
.../yarn/server/resourcemanager/RMAppManager.java | 32 ++++-
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../recovery/records/ApplicationStateData.java | 45 ++++++-
.../impl/pb/ApplicationStateDataPBImpl.java | 17 ++-
.../yarn/server/resourcemanager/rmapp/RMApp.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 25 +++-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/AppManagerTestBase.java | 3 +-
.../yarn/server/resourcemanager/RMHATestBase.java | 4 +-
.../applicationsmanager/MockAsm.java | 4 +
.../server/resourcemanager/rmapp/MockRMApp.java | 5 +
.../scheduler/capacity/TestLeafQueue.java | 145 +++++++++++++++++++++
.../src/site/markdown/CapacityScheduler.md | 4 +-
15 files changed, 284 insertions(+), 20 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 516093c..52864f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -597,10 +597,12 @@ public class ClientRMService extends AbstractService implements
// checked here, those that are dependent on RM configuration are validated
// in RMAppManager.
+ UserGroupInformation userUgi = null;
String user = null;
try {
// Safety
- user = UserGroupInformation.getCurrentUser().getShortUserName();
+ userUgi = UserGroupInformation.getCurrentUser();
+ user = userUgi.getShortUserName();
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@@ -691,7 +693,7 @@ public class ClientRMService extends AbstractService implements
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
- System.currentTimeMillis(), user);
+ System.currentTimeMillis(), userUgi);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index fe18d82..2086034 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -352,16 +352,26 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
}
- @SuppressWarnings("unchecked")
+ @VisibleForTesting
+ @Deprecated
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
String user) throws YarnException {
+ submitApplication(submissionContext, submitTime,
+ UserGroupInformation.createRemoteUser(user));
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ protected void submitApplication(
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ UserGroupInformation userUgi) throws YarnException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Passing start time as -1. It will be eventually set in RMAppImpl
// constructor.
RMAppImpl application = createAndPopulateNewRMApp(
- submissionContext, submitTime, user, false, -1, null);
+ submissionContext, submitTime, userUgi, false, -1, null);
try {
if (UserGroupInformation.isSecurityEnabled()) {
this.rmContext.getDelegationTokenRenewer()
@@ -394,11 +404,21 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
ApplicationSubmissionContext appContext =
appState.getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
+ UserGroupInformation userUgi = null;
+ if (appState.getRealUser() != null) {
+ UserGroupInformation realUserUgi = null;
+ realUserUgi =
+ UserGroupInformation.createRemoteUser(appState.getRealUser());
+ userUgi = UserGroupInformation.createProxyUser(appState.getUser(),
+ realUserUgi);
+ } else {
+ userUgi = UserGroupInformation.createRemoteUser(appState.getUser());
+ }
// create and recover app.
RMAppImpl application =
createAndPopulateNewRMApp(appContext, appState.getSubmitTime(),
- appState.getUser(), true, appState.getStartTime(),
+ userUgi, true, appState.getStartTime(),
appState.getState());
application.handle(new RMAppRecoverEvent(appId, rmState));
@@ -406,8 +426,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user, boolean isRecovery, long startTime,
+ UserGroupInformation userUgi, boolean isRecovery, long startTime,
RMAppState recoveredFinalState) throws YarnException {
+ String user = userUgi.getShortUserName();
ApplicationPlacementContext placementContext = null;
if (recoveredFinalState == null) {
@@ -431,7 +452,6 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Verify and get the update application priority and set back to
// submissionContext
- UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
// Application priority needed to be validated only while submitting. During
// recovery, validated priority could be recovered from submission context.
@@ -503,7 +523,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Create RMApp
RMAppImpl application =
new RMAppImpl(applicationId, rmContext, this.conf,
- submissionContext.getApplicationName(), user,
+ submissionContext.getApplicationName(), userUgi,
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
submitTime, submissionContext.getApplicationType(),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 87f3f0d..7b99c21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -937,7 +937,8 @@ public abstract class RMStateStore extends AbstractService {
assert context instanceof ApplicationSubmissionContextPBImpl;
ApplicationStateData appState =
ApplicationStateData.newInstance(app.getSubmitTime(),
- app.getStartTime(), context, app.getUser(), app.getCallerContext());
+ app.getStartTime(), context, app.getUser(), app.getRealUser(),
+ app.getCallerContext());
appState.setApplicationTimeouts(app.getApplicationTimeouts());
getRMStateStoreEventHandler().handle(new RMStateStoreAppEvent(appState));
}
@@ -1170,7 +1171,7 @@ public abstract class RMStateStore extends AbstractService {
ApplicationStateData appState =
ApplicationStateData.newInstance(app.getSubmitTime(),
app.getStartTime(), app.getApplicationSubmissionContext(),
- app.getUser(), app.getCallerContext());
+ app.getUser(), app.getRealUser(), app.getCallerContext());
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
appState.attempts.put(appAttempt.getAppAttemptId(), null);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
index 2b0bd2b..8b908e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
@@ -92,9 +92,46 @@ public abstract class ApplicationStateData {
public static ApplicationStateData newInstance(long submitTime,
long startTime, ApplicationSubmissionContext context, String user) {
- return newInstance(submitTime, startTime, context, user, null);
+ return newInstance(submitTime, startTime, context, user,
+ (CallerContext) null);
}
-
+
+ public static ApplicationStateData newInstance(long submitTime,
+ long startTime, String user, String realUser,
+ ApplicationSubmissionContext submissionContext, RMAppState state,
+ String diagnostics, long launchTime, long finishTime,
+ CallerContext callerContext) {
+ ApplicationStateData appState =
+ newInstance(submitTime, startTime, user, submissionContext, state,
+ diagnostics, launchTime, finishTime, callerContext);
+ if (realUser != null) {
+ appState.setRealUser(realUser);
+ }
+ return appState;
+ }
+
+ public static ApplicationStateData newInstance(long submitTime,
+ long startTime, String user, String realUser,
+ ApplicationSubmissionContext submissionContext, RMAppState state,
+ String diagnostics, long launchTime, long finishTime,
+ CallerContext callerContext,
+ Map<ApplicationTimeoutType, Long> applicationTimeouts) {
+ ApplicationStateData appState =
+ newInstance(submitTime, startTime, user, submissionContext, state,
+ diagnostics, launchTime, finishTime, callerContext, applicationTimeouts);
+ if (realUser != null) {
+ appState.setRealUser(realUser);
+ }
+ return appState;
+ }
+
+ public static ApplicationStateData newInstance(long submitTime,
+ long startTime, ApplicationSubmissionContext context, String user,
+ String realUser, CallerContext callerContext) {
+ return newInstance(submitTime, startTime, user, realUser, context, null, "",
+ 0, 0, callerContext);
+ }
+
public int getAttemptCount() {
return attempts.size();
}
@@ -213,4 +250,8 @@ public abstract class ApplicationStateData {
@Public
public abstract void setApplicationTimeouts(
Map<ApplicationTimeoutType, Long> applicationTimeouts);
+
+ public abstract String getRealUser();
+
+ public abstract void setRealUser(String realUser);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
index 0f5186e..5dcb70b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
@@ -148,7 +148,22 @@ public class ApplicationStateDataPBImpl extends ApplicationStateData {
maybeInitBuilder();
builder.setUser(user);
}
-
+
+ @Override
+ public String getRealUser() {
+ ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasRealUser()) {
+ return null;
+ }
+ return (p.getRealUser());
+ }
+
+ @Override
+ public void setRealUser(String realUser) {
+ maybeInitBuilder();
+ builder.setRealUser(realUser);
+ }
+
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index d8323f5..949b34e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -325,4 +325,6 @@ public interface RMApp extends EventHandler<RMAppEvent> {
* @return Map of envs related to application scheduling preferences.
*/
Map<String, String> getApplicationSchedulingEnvs();
+
+ String getRealUser();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 23f49b0..8ff03d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -133,6 +133,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private final RMContext rmContext;
private final Configuration conf;
private final String user;
+ private final UserGroupInformation userUgi;
private final String name;
private final ApplicationSubmissionContext submissionContext;
private final Dispatcher dispatcher;
@@ -420,7 +421,19 @@ public class RMAppImpl implements RMApp, Recoverable {
String applicationType, Set<String> applicationTags,
List<ResourceRequest> amReqs, ApplicationPlacementContext
placementContext, long startTime) {
+ this(applicationId, rmContext, config, name,
+ (user != null ? UserGroupInformation.createRemoteUser(user) : null),
+ queue, submissionContext, scheduler, masterService, submitTime,
+ applicationType, applicationTags, amReqs, placementContext, startTime);
+ }
+ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
+ Configuration config, String name, UserGroupInformation userUgi,
+ String queue, ApplicationSubmissionContext submissionContext,
+ YarnScheduler scheduler, ApplicationMasterService masterService,
+ long submitTime, String applicationType, Set<String> applicationTags,
+ List<ResourceRequest> amReqs, ApplicationPlacementContext
+ placementContext, long startTime) {
this.systemClock = SystemClock.getInstance();
this.applicationId = applicationId;
@@ -429,7 +442,9 @@ public class RMAppImpl implements RMApp, Recoverable {
this.dispatcher = rmContext.getDispatcher();
this.handler = dispatcher.getEventHandler();
this.conf = config;
- this.user = StringInterner.weakIntern(user);
+ this.user = StringInterner.weakIntern(
+ (userUgi != null) ? userUgi.getShortUserName() : null);
+ this.userUgi = userUgi;
this.queue = queue;
this.submissionContext = submissionContext;
this.scheduler = scheduler;
@@ -1309,7 +1324,7 @@ public class RMAppImpl implements RMApp, Recoverable {
ApplicationStateData appState =
ApplicationStateData.newInstance(this.submitTime, this.startTime,
- this.user, this.submissionContext,
+ this.getUser(), this.getRealUser(), this.submissionContext,
stateToBeStored, diags, this.launchTime, this.storedFinishTime,
this.callerContext);
appState.setApplicationTimeouts(this.applicationTimeouts);
@@ -1898,4 +1913,10 @@ public class RMAppImpl implements RMApp, Recoverable {
Clock getSystemClock() {
return systemClock;
}
+
+ @Override
+ public String getRealUser() {
+ UserGroupInformation realUserUgi = this.userUgi.getRealUser();
+ return (realUserUgi != null) ? realUserUgi.getShortUserName() : null;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 6f49b85..69e775f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -883,6 +883,10 @@ public class CapacityScheduler extends
} catch (AccessControlException ace) {
// Ignore the exception for recovered app as the app was previously
// accepted.
+ LOG.warn("AccessControlException received when trying to recover "
+ + applicationId + " in queue " + queueName + " for user " + user
+ + ". Since the app was in the queue prior to recovery, the Capacity"
+ + " Scheduler will recover the app anyway.", ace);
}
queue.getMetrics().submitApp(user);
SchedulerApplication<FiCaSchedulerApp> application =
@@ -2777,7 +2781,7 @@ public class CapacityScheduler extends
ApplicationStateData appState = ApplicationStateData.newInstance(
rmApp.getSubmitTime(), rmApp.getStartTime(),
rmApp.getApplicationSubmissionContext(), rmApp.getUser(),
- rmApp.getCallerContext());
+ rmApp.getRealUser(), rmApp.getCallerContext());
appState.setApplicationTimeouts(rmApp.getApplicationTimeouts());
appState.setLaunchTime(rmApp.getLaunchTime());
rmContext.getStateStore().updateApplicationStateSynchronously(appState,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 61f4a8a..2d1cf1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -72,6 +72,7 @@ message ApplicationStateDataProto {
optional hadoop.common.RPCCallerContextProto caller_context = 8;
repeated ApplicationTimeoutMapProto application_timeouts = 9;
optional int64 launch_time = 10;
+ optional string real_user = 11;
}
message ApplicationAttemptStateDataProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
index 33fe028..2e43aa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -71,7 +72,7 @@ public class AppManagerTestBase {
ApplicationSubmissionContext submissionContext, String user)
throws YarnException {
super.submitApplication(submissionContext, System.currentTimeMillis(),
- user);
+ UserGroupInformation.createRemoteUser(user));
}
public String getUserNameForPlacement(final String user,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
index 439a449..ca76c24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -169,7 +170,8 @@ public abstract class RMHATestBase extends ClientBaseWithFixes{
@Override
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user) throws YarnException {
+ UserGroupInformation userUgi) throws YarnException {
+ String user = userUgi.getShortUserName();
//Do nothing, just add the application to RMContext
RMAppImpl application =
new RMAppImpl(submissionContext.getApplicationId(), this.rmContext,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index f8f0d34..97de938 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -69,6 +69,10 @@ public abstract class MockAsm extends MockApps {
}
@Override
+ public String getRealUser() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+ @Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
throw new UnsupportedOperationException("Not supported yet.");
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index ad29d27..96cff0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -119,6 +119,11 @@ public class MockRMApp implements RMApp {
return user;
}
+ @Override
+ public String getRealUser() {
+ return null;
+ }
+
public void setUser(String user) {
this.user = user;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 89024b6..728e7f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -56,11 +56,15 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -72,9 +76,13 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -91,8 +99,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuot
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
@@ -4812,4 +4822,139 @@ public class TestLeafQueue {
cs.stop();
}
}
+
+ private static class TestRMAppManager extends RMAppManager {
+ TestRMAppManager(RMContext context, YarnScheduler scheduler,
+ ApplicationMasterService masterService,
+ ApplicationACLsManager applicationACLsManager, Configuration conf) {
+ super(context, scheduler, masterService, applicationACLsManager, conf);
+ }
+
+ @Override
+ public void submitApplication(
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ UserGroupInformation userUgi) throws YarnException {
+ super.submitApplication(submissionContext, submitTime, userUgi);
+ }
+ }
+
+ @Test
+ public void testSubmitUsingRealUserAcls() throws Exception {
+ final String realUser = "AdminUser";
+ final String user0 = "user0";
+ final String user1 = "user1";
+ final String queue = "default";
+
+ YarnConfiguration conf = new YarnConfiguration();
+ MockRM rm = new MockRM();
+ rm.init(conf);
+ rm.start();
+ rm.getConfig().setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+
+ UserGroupInformation realUserUgi =
+ UserGroupInformation.createRemoteUser(realUser);
+ UserGroupInformation ugi0 =
+ UserGroupInformation.createProxyUserForTesting("user0", realUserUgi,
+ new String[] {"group1"});
+ UserGroupInformation ugi1 =
+ UserGroupInformation.createProxyUserForTesting("user1", realUserUgi,
+ new String[] {"group1"});
+ ApplicationId applicationId0 = TestUtils.getMockApplicationId(0);
+ ApplicationId applicationId1 = TestUtils.getMockApplicationId(1);
+ CapacityScheduler cSched = (CapacityScheduler) rm.getResourceScheduler();
+
+ ParentQueue rootQueue = (ParentQueue) cSched.getRootQueue();
+ Map<AccessType, AccessControlList> rootAcls = rootQueue.acls;
+ rootAcls.put(AccessType.SUBMIT_APP, new AccessControlList(realUser));
+ rootAcls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+
+ LeafQueue defaultQueue = (LeafQueue)cSched.getQueue(queue);
+ Map<AccessType, AccessControlList> a = defaultQueue.acls;
+ a.put(AccessType.SUBMIT_APP, new AccessControlList(realUser));
+ a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+
+ TestRMAppManager testRmAppManager =
+ new TestRMAppManager(rmContext, cSched, rm.getApplicationMasterService(),
+ rm.getApplicationACLsManager(), rm.getConfig());
+ ContainerLaunchContext clc =
+ mock(ContainerLaunchContext.class);
+ ApplicationSubmissionContext asc =
+ ApplicationSubmissionContext.newInstance(
+ applicationId0, "myApp0", "default", Priority.newInstance(0),
+ clc, false, false, 1, Resource.newInstance(1024, 1));
+
+ // Each of the following test cases has a proxied user and a real user.
+ // The proxied users are user0 and user1, depending on the test. The real
+ // user is always AdminUser.
+
+ // Ensure that user0 is not allowed to submit to the default queue when only
+ // the admin user is in the submit ACL and the admin user does not have the
+ // USE_REAL_ACLS character prepended.
+ try {
+ testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0);
+ Assert.fail(user0 + " should not be allowed to submit to the "
+ + queue + " queue when only admin user is in submit ACL.");
+ } catch (YarnException e) {
+ // This is the expected behavior.
+ assertTrue("Should have received an AccessControlException.",
+ e.getCause() instanceof AccessControlException);
+ }
+
+ // With only user0 in the list of users authorized to submit apps to the
+ // queue, ensure that user0 is allowed to submit to the default queue.
+ a.put(AccessType.SUBMIT_APP, new AccessControlList(user0));
+ a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+ try {
+ testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0);
+ } catch (YarnException e) {
+ Assert.fail(user0 + " should be allowed to submit to the "
+ + queue + " queue.");
+ }
+
+ // With only user0 in the list of users authorized to submit apps to the
+ // queue, ensure that user1 is NOT allowed to submit to the default queue
+ try {
+ testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+ Assert.fail(user1 + " should not be allowed to submit to the "
+ + queue + " queue.");
+ } catch (YarnException e) {
+ // This is the expected behavior.
+ assertTrue("Should have received an AccessControlException.",
+ e.getCause() instanceof AccessControlException);
+ }
+
+ // Even though the admin user is in the list of users allowed to submit to
+ // the default queue and user1's real user is the admin user, user1 should
+ // not be allowed to submit to queue because the ACL entry does not have the
+ // special character prepended in the list.
+ a.put(AccessType.SUBMIT_APP,
+ new AccessControlList(user0 + "," + realUser));
+ try {
+ testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+ Assert.fail(user1 + " should not be allowed to submit to the "
+ + queue + " queue.");
+ } catch (YarnException e) {
+ // This is the expected behavior.
+ assertTrue("Should have received an AccessControlException.",
+ e.getCause() instanceof AccessControlException);
+ }
+
+ // user1 should now be allowed to submit to the default queue because the
+ // admin user is in the ACL list and has the USE_REAL_ACLS character
+ // prepended.
+ a.put(AccessType.SUBMIT_APP,
+ new AccessControlList(user0 + ","
+ + AccessControlList.USE_REAL_ACLS + realUser));
+ asc.setApplicationId(applicationId1);
+ try {
+ testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+ } catch (YarnException e) {
+ LOG.error("failed to submit", e);
+ Assert.fail(user1 + " should be allowed to submit to the "
+ + queue + " queue when real user is" + realUser + ".");
+ }
+
+ rm.stop();
+ rm.close();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 169ee3b..9cdb579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -167,8 +167,8 @@ Configuration
| Property | Description |
|:---- |:---- |
| `yarn.scheduler.capacity.<queue-path>.state` | The *state* of the queue. Can be one of `RUNNING` or `STOPPED`. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Thus, if the *root* queue is `STOPPED` no applications can be submitted to the entire cluster. Existing applications continue to completion, thus the queue can be *drained* gracefully. Value is specified as Enumeration. |
-| `yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. |
-| `yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. |
+| `yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to submit to the queue. |
+| `yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to administer [...]
**Note:** An *ACL* is of the form *user1*,*user2* *space* *group1*,*group2*. The special value of * implies *anyone*. The special value of *space* implies *no one*. The default is * for the root queue if not specified.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org