You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/02/14 20:30:54 UTC
svn commit: r1568497 - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yar...
Author: cnauroth
Date: Fri Feb 14 19:30:52 2014
New Revision: 1568497
URL: http://svn.apache.org/r1568497
Log:
Merge trunk to HDFS-4685.
Modified:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt Fri Feb 14 19:30:52 2014
@@ -184,6 +184,10 @@ Release 2.4.0 - UNRELEASED
YARN-1345. Remove FINAL_SAVING state from YarnApplicationAttemptState
(Zhijie Shen via jianhe)
+ YARN-1676. Modified RM HA handling of user-to-group mappings to
+ be available across RM failover by making using of a remote
+ configuration-provider. (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -265,6 +269,10 @@ Release 2.4.0 - UNRELEASED
YARN-1578. Fixed reading incomplete application attempt and container data
in FileSystemApplicationHistoryStore. (Shinichi Yamashita via zjshen)
+ YARN-1417. Modified RM to generate container-tokens not at creation time, but
+ at allocation time so as to prevent RM from shelling out containers with
+ expired tokens. (Omkar Vinit Joshi and Jian He via vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Fri Feb 14 19:30:52 2014
@@ -381,21 +381,22 @@ public class AdminService extends Compos
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
RefreshUserToGroupsMappingsRequest request)
- throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
+ throws YarnException, IOException {
+ String argName = "refreshUserToGroupsMappings";
+ UserGroupInformation user = checkAcls(argName);
- // TODO (YARN-1459): Revisit handling user-groups on Standby RM
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(),
- "refreshUserToGroupsMapping",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh user-groups.");
throwStandbyException();
}
- Groups.getUserToGroupsMappingService().refresh();
- RMAuditLogger.logSuccess(user.getShortUserName(),
- "refreshUserToGroupsMappings", "AdminService");
+ Groups.getUserToGroupsMappingService(
+ getConfiguration(getConfig(),
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();
+
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshUserToGroupsMappingsResponse.class);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Feb 14 19:30:52 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
@@ -198,6 +199,9 @@ public class ResourceManager extends Com
this.conf =
this.configurationProvider.getConfiguration(this.conf,
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
+ .refresh();
}
// register the handlers for all AlwaysOn services using setupDispatcher().
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Fri Feb 14 19:30:52 2014
@@ -345,6 +345,11 @@ public class SchedulerApplicationAttempt
for (RMContainer rmContainer : newlyAllocatedContainers) {
rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
RMContainerEventType.ACQUIRED));
+ Container container = rmContainer.getContainer();
+ rmContainer.getContainer().setContainerToken(
+ rmContext.getContainerTokenSecretManager().createContainerToken(
+ rmContainer.getContainerId(), container.getNodeId(), getUser(),
+ container.getResource()));
returnContainerList.add(rmContainer.getContainer());
}
newlyAllocatedContainers.clear();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Feb 14 19:30:52 2014
@@ -1292,16 +1292,6 @@ public class LeafQueue implements CSQueu
return container;
}
- /**
- * Create <code>ContainerToken</code>, only in secure-mode
- */
- Token createContainerToken(
- FiCaSchedulerApp application, Container container) {
- return containerTokenSecretManager.createContainerToken(
- container.getId(), container.getNodeId(),
- application.getUser(), container.getResource());
- }
-
private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node,
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
@@ -1345,14 +1335,6 @@ public class LeafQueue implements CSQueu
unreserve(application, priority, node, rmContainer);
}
- Token containerToken =
- createContainerToken(application, container);
- if (containerToken == null) {
- // Something went wrong...
- return Resources.none();
- }
- container.setContainerToken(containerToken);
-
// Inform the application
RMContainer allocatedContainer =
application.allocate(type, node, priority, request, container);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Feb 14 19:30:52 2014
@@ -151,17 +151,11 @@ public class AppSchedulable extends Sche
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- org.apache.hadoop.yarn.api.records.Token containerToken =
- containerTokenSecretManager.createContainerToken(containerId, nodeId,
- application.getUser(), capability);
- if (containerToken == null) {
- return null; // Try again later.
- }
// Create the container
Container container =
BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
- .getHttpAddress(), capability, priority, containerToken);
+ .getHttpAddress(), capability, priority, null);
return container;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Feb 14 19:30:52 2014
@@ -654,20 +654,11 @@ public class FifoScheduler extends Abstr
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- Token containerToken = null;
-
- containerToken =
- this.rmContext.getContainerTokenSecretManager()
- .createContainerToken(containerId, nodeId, application.getUser(),
- capability);
- if (containerToken == null) {
- return i; // Try again later.
- }
// Create the container
Container container =
BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
- .getHttpAddress(), capability, priority, containerToken);
+ .getHttpAddress(), capability, priority, null);
// Allocate!
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Feb 14 19:30:52 2014
@@ -142,8 +142,15 @@ public class MockRM extends ResourceMana
public void waitForState(MockNM nm, ContainerId containerId,
RMContainerState containerState) throws Exception {
RMContainer container = getResourceScheduler().getRMContainer(containerId);
- Assert.assertNotNull("Container shouldn't be null", container);
int timeoutSecs = 0;
+ while(container == null && timeoutSecs++ < 20) {
+ nm.nodeHeartbeat(true);
+ container = getResourceScheduler().getRMContainer(containerId);
+ System.out.println("Waiting for container " + containerId + " to be allocated.");
+ Thread.sleep(100);
+ }
+ Assert.assertNotNull("Container shouldn't be null", container);
+ timeoutSecs = 0;
while (!containerState.equals(container.getState()) && timeoutSecs++ < 40) {
System.out.println("Container : " + containerId + " State is : "
+ container.getState() + " Waiting for state : " + containerState);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Fri Feb 14 19:30:52 2014
@@ -24,10 +24,17 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
@@ -37,6 +44,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
@@ -366,6 +374,84 @@ public class TestRMAdminService {
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
}
+ @Test
+ public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ try {
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ } catch (Exception ex) {
+ fail("Using localConfigurationProvider. Should not get any exception.");
+ }
+ }
+
+ @Test
+ public void
+ testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch (Exception ex) {
+ // Expect exception here
+ }
+
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ List<String> groupWithInit =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+
+ // upload default configurations
+ uploadDefaultConfiguration();
+ Configuration conf = new Configuration();
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ MockUnixGroupsMapping.class,
+ GroupMappingServiceProvider.class);
+ uploadConfiguration(conf, "core-site.xml");
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ // Make sure RM will use the updated GroupMappingServiceProvider
+ List<String> groupBefore =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+ Assert.assertTrue(groupBefore.contains("test_group_A")
+ && groupBefore.contains("test_group_B")
+ && groupBefore.contains("test_group_C") && groupBefore.size() == 3);
+ Assert.assertTrue(groupWithInit.size() != groupBefore.size());
+ Assert.assertFalse(groupWithInit.contains("test_group_A")
+ || groupWithInit.contains("test_group_B")
+ || groupWithInit.contains("test_group_C"));
+
+ // update the groups
+ MockUnixGroupsMapping.updateGroups();
+
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ List<String> groupAfter =
+ Groups.getUserToGroupsMappingService(configuration).getGroups(user);
+
+ // should get the updated groups
+ Assert.assertTrue(groupAfter.contains("test_group_D")
+ && groupAfter.contains("test_group_E")
+ && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
+
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
@@ -418,4 +504,38 @@ public class TestRMAdminService {
.addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
}
+
+ private static class MockUnixGroupsMapping implements
+ GroupMappingServiceProvider {
+
+ @SuppressWarnings("serial")
+ private static List<String> group = new ArrayList<String>() {{
+ add("test_group_A");
+ add("test_group_B");
+ add("test_group_C");
+ }};
+
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ return group;
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ // Do nothing
+ }
+
+ public static void updateGroups() {
+ group.clear();
+ group.add("test_group_D");
+ group.add("test_group_E");
+ group.add("test_group_F");
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java?rev=1568497&r1=1568496&r2=1568497&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java Fri Feb 14 19:30:52 2014
@@ -18,11 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.ArrayList;
+import java.util.List;
+
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -30,6 +36,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.junit.Test;
@@ -106,4 +115,38 @@ public class TestContainerAllocation {
rm.stop();
}
+
+ // This is to test container tokens are generated when the containers are
+ // acquired by the AM, not when the containers are allocated
+ @Test
+ public void testContainerTokenGeneratedOnPullRequest() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
+ RMApp app1 = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ // request a container.
+ am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
+ ContainerId containerId2 =
+ ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
+ rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
+
+ RMContainer container =
+ rm1.getResourceScheduler().getRMContainer(containerId2);
+ // no container token is generated.
+ Assert.assertEquals(containerId2, container.getContainerId());
+ Assert.assertNull(container.getContainer().getContainerToken());
+
+ // acquire the container.
+ List<Container> containers =
+ am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ Assert.assertEquals(containerId2, containers.get(0).getId());
+ // container token is generated.
+ Assert.assertNotNull(containers.get(0).getContainerToken());
+ rm1.stop();
+ }
}
\ No newline at end of file