You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/09/15 16:56:12 UTC
[02/50] [abbrv] hadoop git commit: YARN-7157. Add admin configuration
to filter per-user's apps in secure cluster. Contributed by Sunil G.
YARN-7157. Add admin configuration to filter per-user's apps in secure cluster. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5324388c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5324388c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5324388c
Branch: refs/heads/YARN-1011
Commit: 5324388cf2357b1f80efd0c34392f577bf417455
Parents: 8277fab
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Sep 13 23:36:39 2017 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Sep 13 23:36:47 2017 +0530
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +++
.../src/main/resources/yarn-default.xml | 10 +++
.../server/resourcemanager/ClientRMService.java | 19 ++++-
.../resourcemanager/TestClientRMService.java | 74 ++++++++++++++++++++
4 files changed, 110 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5324388c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c4e8ff2..c6ec6fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3158,6 +3158,14 @@ public class YarnConfiguration extends Configuration {
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
+ /*
+ * Support to view apps for given user in secure cluster.
+ */
+ public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
+ RM_PREFIX + "display.per-user-apps";
+ public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
+ false;
+
// RM and NM CSRF props
public static final String REST_CSRF = "webapp.rest-csrf.";
public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5324388c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 18f790d..d16d956 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3323,4 +3323,14 @@
to specify details about the individual resource types.
</description>
</property>
+
+ <property>
+ <name>yarn.resourcemanager.display.per-user-apps</name>
+ <value>false</value>
+ <description>
+ Flag to enable display of applications per user as an admin
+ configuration.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5324388c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index d98e558..a0c3db6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -215,6 +215,8 @@ public class ClientRMService extends AbstractService implements
private ReservationSystem reservationSystem;
private ReservationInputValidator rValidator;
+ private boolean displayPerUserApps = false;
+
private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
RMAppState.ACCEPTED, RMAppState.RUNNING);
@@ -275,7 +277,11 @@ public class ClientRMService extends AbstractService implements
}
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
-
+
+ this.displayPerUserApps = conf.getBoolean(
+ YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
+ YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+
this.server.start();
clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_ADDRESS,
@@ -909,6 +915,12 @@ public class ClientRMService extends AbstractService implements
continue;
}
+ // Given RM is configured to display apps per user, skip apps to which
+ // this caller doesn't have access to view.
+ if (displayPerUserApps && !allowAccess) {
+ continue;
+ }
+
reports.add(application.createAndGetApplicationReport(
callerUGI.getUserName(), allowAccess));
}
@@ -1804,4 +1816,9 @@ public class ClientRMService extends AbstractService implements
response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
+
+ @VisibleForTesting
+ public void setDisplayPerUserApps(boolean displayPerUserApps) {
+ this.displayPerUserApps = displayPerUserApps;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5324388c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 49718a2..35b3f86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1121,6 +1121,12 @@ public class TestClientRMService {
assertEquals("Incorrect number of applications for user", 3,
rmService.getApplications(request).getApplicationList().size());
+ rmService.setDisplayPerUserApps(true);
+ userSet.clear();
+ assertEquals("Incorrect number of applications for user", 6,
+ rmService.getApplications(request).getApplicationList().size());
+ rmService.setDisplayPerUserApps(false);
+
// Check tags
request = GetApplicationsRequest.newInstance(
ApplicationsRequestScope.ALL, null, null, null, null, null, null,
@@ -2107,4 +2113,72 @@ public class TestClientRMService {
rm.stop();
rpc.stopProxy(client, conf);
}
+
+ @Test
+ public void testGetApplicationsWithPerUserApps()
+ throws IOException, YarnException {
+ /*
+ * Submit 3 applications alternately in two queues
+ */
+ // Basic setup
+ YarnScheduler yarnScheduler = mockYarnScheduler();
+ RMContext rmContext = mock(RMContext.class);
+ mockRMContext(yarnScheduler, rmContext);
+ RMStateStore stateStore = mock(RMStateStore.class);
+ when(rmContext.getStateStore()).thenReturn(stateStore);
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
+ RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
+ mock(ApplicationACLsManager.class), new Configuration());
+ when(rmContext.getDispatcher().getEventHandler())
+ .thenReturn(new EventHandler<Event>() {
+ public void handle(Event event) {
+ }
+ });
+
+ // Simulate Queue ACL manager which returns false always
+ QueueACLsManager queueAclsManager = mock(QueueACLsManager.class);
+ when(queueAclsManager.checkAccess(any(UserGroupInformation.class),
+ any(QueueACL.class), any(RMApp.class), any(String.class),
+ anyListOf(String.class))).thenReturn(false);
+
+ // Simulate app ACL manager which returns false always
+ ApplicationACLsManager appAclsManager = mock(ApplicationACLsManager.class);
+ when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()),
+ any(ApplicationAccessType.class), any(String.class),
+ any(ApplicationId.class))).thenReturn(false);
+ ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+ appManager, appAclsManager, queueAclsManager, null);
+ rmService.init(new Configuration());
+
+ // Initialize appnames and queues
+ String[] queues = {QUEUE_1, QUEUE_2};
+ String[] appNames = {MockApps.newAppName(), MockApps.newAppName(),
+ MockApps.newAppName()};
+ ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102),
+ getApplicationId(103)};
+ List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
+
+ long[] submitTimeMillis = new long[3];
+ // Submit applications
+ for (int i = 0; i < appIds.length; i++) {
+ ApplicationId appId = appIds[i];
+ SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId,
+ appNames[i], queues[i % queues.length],
+ new HashSet<String>(tags.subList(0, i + 1)));
+ rmService.submitApplication(submitRequest);
+ submitTimeMillis[i] = System.currentTimeMillis();
+ }
+
+ // Test different cases of ClientRMService#getApplications()
+ GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+ assertEquals("Incorrect total number of apps", 6,
+ rmService.getApplications(request).getApplicationList().size());
+
+ rmService.setDisplayPerUserApps(true);
+ assertEquals("Incorrect number of applications for user", 0,
+ rmService.getApplications(request).getApplicationList().size());
+ rmService.setDisplayPerUserApps(false);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org