You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/02/14 19:32:46 UTC
svn commit: r1568437 [2/2] - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop...
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Feb 14 18:32:37 2014
@@ -157,6 +157,7 @@ public class RMAppAttemptImpl implements
private RMAppAttemptEvent eventCausingFinalSaving;
private RMAppAttemptState targetedFinalState;
private RMAppAttemptState recoveredFinalState;
+ private RMAppAttemptState stateBeforeFinalSaving;
private Object transitionTodo;
private static final StateMachineFactory<RMAppAttemptImpl,
@@ -885,6 +886,7 @@ public class RMAppAttemptImpl implements
RMAppAttemptState stateToBeStored) {
rememberTargetTransitions(event, transitionToDo, targetFinalState);
+ stateBeforeFinalSaving = getState();
// As of today, finalState, diagnostics, final-tracking-url and
// finalAppStatus are the only things that we store into the StateStore
@@ -1536,6 +1538,10 @@ public class RMAppAttemptImpl implements
@Override
public YarnApplicationAttemptState createApplicationAttemptState() {
RMAppAttemptState state = getState();
+ // If AppAttempt is in FINAL_SAVING state, return its previous state.
+ if (state.equals(RMAppAttemptState.FINAL_SAVING)) {
+ state = stateBeforeFinalSaving;
+ }
return RMServerUtils.createApplicationAttemptState(state);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Fri Feb 14 18:32:37 2014
@@ -150,7 +150,7 @@ public class SchedulerApplicationAttempt
return appSchedulingInfo.getPriorities();
}
- public ResourceRequest getResourceRequest(Priority priority, String resourceName) {
+ public synchronized ResourceRequest getResourceRequest(Priority priority, String resourceName) {
return this.appSchedulingInfo.getResourceRequest(priority, resourceName);
}
@@ -158,7 +158,7 @@ public class SchedulerApplicationAttempt
return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers();
}
- public Resource getResource(Priority priority) {
+ public synchronized Resource getResource(Priority priority) {
return appSchedulingInfo.getResource(priority);
}
@@ -345,6 +345,11 @@ public class SchedulerApplicationAttempt
for (RMContainer rmContainer : newlyAllocatedContainers) {
rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
RMContainerEventType.ACQUIRED));
+ Container container = rmContainer.getContainer();
+ rmContainer.getContainer().setContainerToken(
+ rmContext.getContainerTokenSecretManager().createContainerToken(
+ rmContainer.getContainerId(), container.getNodeId(), getUser(),
+ container.getResource()));
returnContainerList.add(rmContainer.getContainer());
}
newlyAllocatedContainers.clear();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Feb 14 18:32:37 2014
@@ -196,7 +196,6 @@ public class CapacityScheduler extends A
private ResourceCalculator calculator;
private boolean usePortForNodeName;
- private boolean useLocalConfigurationProvider;
public CapacityScheduler() {}
@@ -262,14 +261,21 @@ public class CapacityScheduler extends A
@Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
+ Configuration configuration = new Configuration(conf);
if (!initialized) {
- this.useLocalConfigurationProvider =
- (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
- YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- LocalConfigurationProvider.class)));
- this.conf =
- new CapacitySchedulerConfiguration(conf,
- this.useLocalConfigurationProvider);
+ if (rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider) {
+ this.conf = new CapacitySchedulerConfiguration(configuration, true);
+ } else {
+ try {
+ this.conf =
+ new CapacitySchedulerConfiguration(rmContext
+ .getConfigurationProvider().getConfiguration(configuration,
+ YarnConfiguration.CS_CONFIGURATION_FILE), false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
@@ -290,7 +296,8 @@ public class CapacityScheduler extends A
CapacitySchedulerConfiguration oldConf = this.conf;
this.conf =
new CapacitySchedulerConfiguration(conf,
- this.useLocalConfigurationProvider);
+ rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -316,6 +323,7 @@ public class CapacityScheduler extends A
@Lock(CapacityScheduler.class)
private void initializeQueues(CapacitySchedulerConfiguration conf)
throws IOException {
+
root =
parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT,
queues, queues, noop);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Feb 14 18:32:37 2014
@@ -1292,16 +1292,6 @@ public class LeafQueue implements CSQueu
return container;
}
- /**
- * Create <code>ContainerToken</code>, only in secure-mode
- */
- Token createContainerToken(
- FiCaSchedulerApp application, Container container) {
- return containerTokenSecretManager.createContainerToken(
- container.getId(), container.getNodeId(),
- application.getUser(), container.getResource());
- }
-
private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node,
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
@@ -1345,14 +1335,6 @@ public class LeafQueue implements CSQueu
unreserve(application, priority, node, rmContainer);
}
- Token containerToken =
- createContainerToken(application, container);
- if (containerToken == null) {
- // Something went wrong...
- return Resources.none();
- }
- container.setContainerToken(containerToken);
-
// Inform the application
RMContainer allocatedContainer =
application.allocate(type, node, priority, request, container);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java Fri Feb 14 18:32:37 2014
@@ -192,7 +192,7 @@ public class FiCaSchedulerApp extends Sc
return Math.min(((float)requiredResources / clusterNodes), 1.0f);
}
- public Resource getTotalPendingRequests() {
+ public synchronized Resource getTotalPendingRequests() {
Resource ret = Resource.newInstance(0, 0);
for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) {
// to avoid double counting we count only "ANY" resource requests
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Feb 14 18:32:37 2014
@@ -82,10 +82,12 @@ public class AppSchedulable extends Sche
Resources.addTo(demand, app.getCurrentConsumption());
// Add up outstanding resource requests
- for (Priority p : app.getPriorities()) {
- for (ResourceRequest r : app.getResourceRequests(p).values()) {
- Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
- Resources.addTo(demand, total);
+ synchronized (app) {
+ for (Priority p : app.getPriorities()) {
+ for (ResourceRequest r : app.getResourceRequests(p).values()) {
+ Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
+ Resources.addTo(demand, total);
+ }
}
}
}
@@ -149,17 +151,11 @@ public class AppSchedulable extends Sche
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- org.apache.hadoop.yarn.api.records.Token containerToken =
- containerTokenSecretManager.createContainerToken(containerId, nodeId,
- application.getUser(), capability);
- if (containerToken == null) {
- return null; // Try again later.
- }
// Create the container
Container container =
BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
- .getHttpAddress(), capability, priority, containerToken);
+ .getHttpAddress(), capability, priority, null);
return container;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Feb 14 18:32:37 2014
@@ -654,20 +654,11 @@ public class FifoScheduler extends Abstr
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- Token containerToken = null;
-
- containerToken =
- this.rmContext.getContainerTokenSecretManager()
- .createContainerToken(containerId, nodeId, application.getUser(),
- capability);
- if (containerToken == null) {
- return i; // Try again later.
- }
// Create the container
Container container =
BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
- .getHttpAddress(), capability, priority, containerToken);
+ .getHttpAddress(), capability, priority, null);
// Allocate!
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java Fri Feb 14 18:32:37 2014
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.security.authorize;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.security.authorize.PolicyProvider;
@@ -37,6 +39,23 @@ import org.apache.hadoop.yarn.server.api
@InterfaceStability.Unstable
public class RMPolicyProvider extends PolicyProvider {
+ private static RMPolicyProvider rmPolicyProvider = null;
+
+ private RMPolicyProvider() {}
+
+ @Private
+ @Unstable
+ public static RMPolicyProvider getInstance() {
+ if (rmPolicyProvider == null) {
+ synchronized(RMPolicyProvider.class) {
+ if (rmPolicyProvider == null) {
+ rmPolicyProvider = new RMPolicyProvider();
+ }
+ }
+ }
+ return rmPolicyProvider;
+ }
+
private static final Service[] resourceManagerServices =
new Service[] {
new Service(
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Feb 14 18:32:37 2014
@@ -142,8 +142,15 @@ public class MockRM extends ResourceMana
public void waitForState(MockNM nm, ContainerId containerId,
RMContainerState containerState) throws Exception {
RMContainer container = getResourceScheduler().getRMContainer(containerId);
- Assert.assertNotNull("Container shouldn't be null", container);
int timeoutSecs = 0;
+ while(container == null && timeoutSecs++ < 20) {
+ nm.nodeHeartbeat(true);
+ container = getResourceScheduler().getRMContainer(containerId);
+ System.out.println("Waiting for container " + containerId + " to be allocated.");
+ Thread.sleep(100);
+ }
+ Assert.assertNotNull("Container shouldn't be null", container);
+ timeoutSecs = 0;
while (!containerState.equals(container.getState()) && timeoutSecs++ < 40) {
System.out.println("Container : " + containerId + " State is : "
+ container.getState() + " Waiting for state : " + containerState);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Fri Feb 14 18:32:37 2014
@@ -24,11 +24,17 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
@@ -38,6 +44,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
@@ -105,34 +112,34 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch(Exception ex) {
+ // Expect exception here
+ }
- // clean the remoteDirectory
- cleanRemoteDirectory();
+ //upload default configurations
+ uploadDefaultConfiguration();
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch(Exception ex) {
+ fail("Should not get any exceptions");
+ }
CapacityScheduler cs =
(CapacityScheduler) rm.getRMContext().getScheduler();
int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();
- try {
- rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
- fail("FileSystemBasedConfigurationProvider is used." +
- " Should get an exception here");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().contains(
- "Can not find Configuration: capacity-scheduler.xml"));
- }
-
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
- String csConfFile = writeConfigurationXML(csConf,
- "capacity-scheduler.xml");
-
- // upload the file into Remote File System
- uploadToRemoteFileSystem(new Path(csConfFile));
+ uploadConfiguration(csConf, "capacity-scheduler.xml");
rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
@@ -159,20 +166,24 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch(Exception ex) {
+ // Expect exception here
+ }
- // clean the remoteDirectory
- cleanRemoteDirectory();
+ //upload default configurations
+ uploadDefaultConfiguration();
try {
- rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance());
- fail("FileSystemBasedConfigurationProvider is used." +
- " Should get an exception here");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().contains(
- "Can not find Configuration: yarn-site.xml"));
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch(Exception ex) {
+ fail("Should not get any exceptions");
}
String aclStringBefore =
@@ -180,10 +191,8 @@ public class TestRMAdminService {
YarnConfiguration yarnConf = new YarnConfiguration();
yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
- String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml");
+ uploadConfiguration(yarnConf, "yarn-site.xml");
- // upload the file into Remote File System
- uploadToRemoteFileSystem(new Path(yarnConfFile));
rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance());
String aclStringAfter =
@@ -214,7 +223,6 @@ public class TestRMAdminService {
}
}
- @SuppressWarnings("resource")
@Test
public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider()
throws IOException, YarnException {
@@ -224,33 +232,33 @@ public class TestRMAdminService {
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
ResourceManager resourceManager = null;
try {
- resourceManager = new ResourceManager();
- resourceManager.init(configuration);
- resourceManager.start();
-
- // clean the remoteDirectory
- cleanRemoteDirectory();
-
try {
- resourceManager.adminService
- .refreshServiceAcls(RefreshServiceAclsRequest
- .newInstance());
- fail("FileSystemBasedConfigurationProvider is used." +
- " Should get an exception here");
+ resourceManager = new ResourceManager();
+ resourceManager.init(configuration);
+ resourceManager.start();
+ fail("Should throw an exception");
} catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().contains(
- "Can not find Configuration: hadoop-policy.xml"));
+ // expect to get an exception here
}
- String aclsString = "alice,bob users,wheel";
+ //upload default configurations
+ uploadDefaultConfiguration();
Configuration conf = new Configuration();
conf.setBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
- conf.set("security.applicationclient.protocol.acl", aclsString);
- String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml");
+ uploadConfiguration(conf, "core-site.xml");
+ try {
+ resourceManager = new ResourceManager();
+ resourceManager.init(configuration);
+ resourceManager.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
- // upload the file into Remote File System
- uploadToRemoteFileSystem(new Path(hadoopConfFile));
+ String aclsString = "alice,bob users,wheel";
+ Configuration newConf = new Configuration();
+ newConf.set("security.applicationclient.protocol.acl", aclsString);
+ uploadConfiguration(newConf, "hadoop-policy.xml");
resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest
.newInstance());
@@ -328,31 +336,31 @@ public class TestRMAdminService {
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
- rm = new MockRM(configuration);
- rm.init(configuration);
- rm.start();
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch(Exception ex) {
+ // Expect exception here
+ }
- // clean the remoteDirectory
- cleanRemoteDirectory();
+ //upload default configurations
+ uploadDefaultConfiguration();
try {
- rm.adminService.refreshSuperUserGroupsConfiguration(
- RefreshSuperUserGroupsConfigurationRequest.newInstance());
- fail("FileSystemBasedConfigurationProvider is used." +
- " Should get an exception here");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().contains(
- "Can not find Configuration: core-site.xml"));
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch(Exception ex) {
+ fail("Should not get any exceptions");
}
Configuration coreConf = new Configuration(false);
coreConf.set("hadoop.proxyuser.test.groups", "test_groups");
coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts");
- String coreConfFile = writeConfigurationXML(coreConf,
- "core-site.xml");
+ uploadConfiguration(coreConf, "core-site.xml");
- // upload the file into Remote File System
- uploadToRemoteFileSystem(new Path(coreConfFile));
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
Assert.assertTrue(ProxyUsers.getProxyGroups()
@@ -366,6 +374,84 @@ public class TestRMAdminService {
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
}
+ @Test
+ public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ try {
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ } catch (Exception ex) {
+ fail("Using localConfigurationProvider. Should not get any exception.");
+ }
+ }
+
+ @Test
+ public void
+ testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ fail("Should throw an exception");
+ } catch (Exception ex) {
+ // Expect exception here
+ }
+
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ List<String> groupWithInit =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+
+ // upload default configurations
+ uploadDefaultConfiguration();
+ Configuration conf = new Configuration();
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ MockUnixGroupsMapping.class,
+ GroupMappingServiceProvider.class);
+ uploadConfiguration(conf, "core-site.xml");
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ // Make sure RM will use the updated GroupMappingServiceProvider
+ List<String> groupBefore =
+ new ArrayList<String>(Groups.getUserToGroupsMappingService(
+ configuration).getGroups(user));
+ Assert.assertTrue(groupBefore.contains("test_group_A")
+ && groupBefore.contains("test_group_B")
+ && groupBefore.contains("test_group_C") && groupBefore.size() == 3);
+ Assert.assertTrue(groupWithInit.size() != groupBefore.size());
+ Assert.assertFalse(groupWithInit.contains("test_group_A")
+ || groupWithInit.contains("test_group_B")
+ || groupWithInit.contains("test_group_C"));
+
+ // update the groups
+ MockUnixGroupsMapping.updateGroups();
+
+ rm.adminService
+ .refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
+ .newInstance());
+ List<String> groupAfter =
+ Groups.getUserToGroupsMappingService(configuration).getGroups(user);
+
+ // should get the updated groups
+ Assert.assertTrue(groupAfter.contains("test_group_D")
+ && groupAfter.contains("test_group_E")
+ && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
+
+ }
+
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;
@@ -393,11 +479,63 @@ public class TestRMAdminService {
fs.copyFromLocalFile(filePath, workingPath);
}
- private void cleanRemoteDirectory() throws IOException {
- if (fs.exists(workingPath)) {
- for (FileStatus file : fs.listStatus(workingPath)) {
- fs.delete(file.getPath(), true);
- }
+ private void uploadConfiguration(Configuration conf, String confFileName)
+ throws IOException {
+ String csConfFile = writeConfigurationXML(conf, confFileName);
+ // upload the file into Remote File System
+ uploadToRemoteFileSystem(new Path(csConfFile));
+ }
+
+ private void uploadDefaultConfiguration() throws IOException {
+ Configuration conf = new Configuration();
+ uploadConfiguration(conf, "core-site.xml");
+
+ YarnConfiguration yarnConf = new YarnConfiguration();
+ yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+ uploadConfiguration(yarnConf, "yarn-site.xml");
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ uploadConfiguration(csConf, "capacity-scheduler.xml");
+
+ Configuration hadoopPolicyConf = new Configuration(false);
+ hadoopPolicyConf
+ .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
+ }
+
+ private static class MockUnixGroupsMapping implements
+ GroupMappingServiceProvider {
+
+ @SuppressWarnings("serial")
+ private static List<String> group = new ArrayList<String>() {{
+ add("test_group_A");
+ add("test_group_B");
+ add("test_group_C");
+ }};
+
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ return group;
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ // Do nothing
+ }
+
+ public static void updateGroups() {
+ group.clear();
+ group.add("test_group_D");
+ group.add("test_group_E");
+ group.add("test_group_F");
}
}
+
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Fri Feb 14 18:32:37 2014
@@ -23,10 +23,7 @@ import static org.junit.Assert.assertTru
import static org.mockito.Mockito.mock;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,15 +31,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
@@ -54,6 +44,7 @@ import org.junit.Test;
public class TestZKRMStateStore extends RMStateStoreTestBase {
public static final Log LOG = LogFactory.getLog(TestZKRMStateStore.class);
+ private static final int ZK_TIMEOUT_MS = 1000;
class TestZKRMStateStoreTester implements RMStateStoreHelper {
@@ -141,6 +132,7 @@ public class TestZKRMStateStore extends
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
conf.set(YarnConfiguration.RM_HA_ID, rmId);
for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
for (String id : HAUtil.getRMHAIds(conf)) {
@@ -182,26 +174,7 @@ public class TestZKRMStateStore extends
HAServiceProtocol.HAServiceState.ACTIVE,
rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
- // Submitting an application to RM1 to trigger a state store operation.
- // RM1 should realize that it got fenced and is not the Active RM anymore.
- Map mockMap = mock(Map.class);
- ApplicationSubmissionContext asc =
- ApplicationSubmissionContext.newInstance(
- ApplicationId.newInstance(1000, 1),
- "testApplication", // app Name
- "default", // queue name
- Priority.newInstance(0),
- ContainerLaunchContext.newInstance(mockMap, mockMap,
- new ArrayList<String>(), mockMap, mock(ByteBuffer.class),
- mockMap),
- false, // unmanaged AM
- true, // cancelTokens
- 1, // max app attempts
- Resource.newInstance(1024, 1));
- ClientRMService rmService = rm1.getClientRMService();
- rmService.submitApplication(SubmitApplicationRequest.newInstance(asc));
-
- for (int i = 0; i < 30; i++) {
+ for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
if (HAServiceProtocol.HAServiceState.ACTIVE ==
rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) {
Thread.sleep(100);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Fri Feb 14 18:32:37 2014
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
@@ -695,6 +696,8 @@ public class TestRMAppAttemptTransitions
// launch AM and verify attempt failed
applicationAttempt.handle(new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl"));
+ assertEquals(YarnApplicationAttemptState.SUBMITTED,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptSubmittedToFailedState(
"Unmanaged AM must register after AM attempt reaches LAUNCHED state.");
}
@@ -710,6 +713,8 @@ public class TestRMAppAttemptTransitions
// launch AM and verify attempt failed
applicationAttempt.handle(new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl"));
+ assertEquals(YarnApplicationAttemptState.SUBMITTED,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertFalse(transferStateFromPreviousAttempt);
}
@@ -720,6 +725,8 @@ public class TestRMAppAttemptTransitions
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.NEW,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS);
verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@@ -740,6 +747,8 @@ public class TestRMAppAttemptTransitions
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.SUBMITTED,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS);
}
@@ -750,6 +759,8 @@ public class TestRMAppAttemptTransitions
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.SCHEDULED,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS);
}
@@ -760,6 +771,8 @@ public class TestRMAppAttemptTransitions
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.ALLOCATED,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptKilledState(amContainer, EMPTY_DIAGNOSTICS);
}
@@ -771,6 +784,8 @@ public class TestRMAppAttemptTransitions
new RMAppAttemptLaunchFailedEvent(
applicationAttempt.getAppAttemptId(),
diagnostics));
+ assertEquals(YarnApplicationAttemptState.ALLOCATED,
+ applicationAttempt.createApplicationAttemptState());
testAppAttemptFailedState(amContainer, diagnostics);
}
@@ -784,6 +799,8 @@ public class TestRMAppAttemptTransitions
ContainerState.COMPLETE, containerDiagMsg, exitCode);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), cs));
+ assertEquals(YarnApplicationAttemptState.ALLOCATED,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
@@ -815,7 +832,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
-
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
@@ -850,7 +868,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
-
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.KILLED,
applicationAttempt.getAppAttemptState());
@@ -871,6 +890,8 @@ public class TestRMAppAttemptTransitions
launchApplicationAttempt(amContainer);
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
+ assertEquals(YarnApplicationAttemptState.LAUNCHED,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
@@ -890,6 +911,8 @@ public class TestRMAppAttemptTransitions
runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
@@ -1055,6 +1078,8 @@ public class TestRMAppAttemptTransitions
diagnostics));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
// Container_finished event comes before Attempt_Saved event.
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
@@ -1083,6 +1108,8 @@ public class TestRMAppAttemptTransitions
diagnostics));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
// Expire event comes before Attempt_saved event.
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
@@ -1118,6 +1145,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.KILL));
+ assertEquals(YarnApplicationAttemptState.LAUNCHED,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
// after attempt is killed, can not get Client Token
token = applicationAttempt.createClientToken(null);
@@ -1140,6 +1169,8 @@ public class TestRMAppAttemptTransitions
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
appAttemptId, cs1));
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
@@ -1178,6 +1209,8 @@ public class TestRMAppAttemptTransitions
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
appAttemptId, cs1));
+ assertEquals(YarnApplicationAttemptState.RUNNING,
+ applicationAttempt.createApplicationAttemptState());
sendAttemptUpdateSavedEvent(applicationAttempt);
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Fri Feb 14 18:32:37 2014
@@ -40,6 +40,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -104,6 +105,7 @@ public class TestCapacityScheduler {
private static float B3_CAPACITY = 20;
private ResourceManager resourceManager = null;
+ private RMContext mockContext;
@Before
public void setUp() throws Exception {
@@ -118,6 +120,9 @@ public class TestCapacityScheduler {
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMNMTokenSecretManager().rollMasterKey();
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ mockContext = mock(RMContext.class);
+ when(mockContext.getConfigurationProvider()).thenReturn(
+ new LocalConfigurationProvider());
}
@After
@@ -133,7 +138,7 @@ public class TestCapacityScheduler {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, null);
+ scheduler.reinitialize(conf, mockContext);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
@@ -147,7 +152,7 @@ public class TestCapacityScheduler {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1);
try {
- scheduler.reinitialize(conf, null);
+ scheduler.reinitialize(conf, mockContext);
fail("Exception is expected because the min vcores allocation is" +
" larger than the max vcores allocation.");
} catch (YarnRuntimeException e) {
@@ -353,7 +358,7 @@ public class TestCapacityScheduler {
conf.setCapacity(A, 80f);
conf.setCapacity(B, 20f);
- cs.reinitialize(conf,null);
+ cs.reinitialize(conf, mockContext);
checkQueueCapacities(cs, 80f, 20f);
}
@@ -503,7 +508,7 @@ public class TestCapacityScheduler {
conf.setCapacity(B2, B2_CAPACITY);
conf.setCapacity(B3, B3_CAPACITY);
conf.setCapacity(B4, B4_CAPACITY);
- cs.reinitialize(conf,null);
+ cs.reinitialize(conf,mockContext);
checkQueueCapacities(cs, 80f, 20f);
// Verify parent for B4
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java Fri Feb 14 18:32:37 2014
@@ -18,11 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.ArrayList;
+import java.util.List;
+
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -30,6 +36,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.junit.Test;
@@ -106,4 +115,38 @@ public class TestContainerAllocation {
rm.stop();
}
+
+ // This is to test container tokens are generated when the containers are
+ // acquired by the AM, not when the containers are allocated
+ @Test
+ public void testContainerTokenGeneratedOnPullRequest() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
+ RMApp app1 = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ // request a container.
+ am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
+ ContainerId containerId2 =
+ ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
+ rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
+
+ RMContainer container =
+ rm1.getResourceScheduler().getRMContainer(containerId2);
+ // no container token is generated.
+ Assert.assertEquals(containerId2, container.getContainerId());
+ Assert.assertNull(container.getContainer().getContainerToken());
+
+ // acquire the container.
+ List<Container> containers =
+ am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ Assert.assertEquals(containerId2, containers.get(0).getId());
+ // container token is generated.
+ Assert.assertNotNull(containers.get(0).getContainerToken());
+ rm1.stop();
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm Fri Feb 14 18:32:37 2014
@@ -35,7 +35,7 @@ Usage: yarn [--config confdir] COMMAND
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| --config confdir | Overwrites the default Configuration directory. Default is ${HADOOP_PREFIX}/conf. |
+| --config confdir | Overwrites the default Configuration directory. Default is $\{HADOOP_PREFIX\}/conf. |
*---------------+--------------+
| COMMAND COMMAND_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into {{User Commands}} and {{Administration Commands}}. |
*---------------+--------------+
@@ -63,11 +63,22 @@ Usage: yarn [--config confdir] COMMAND
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -status ApplicationId | Specify an application id |
+| -list | Lists applications from the RM. Supports optional use of -appTypes
+| | to filter applications based on application type, and -appStates to
+| | filter applications based on application state.
+*---------------+--------------+
+| -appStates States | Works with -list to filter applications based on input
+| | comma-separated list of application states. The valid
+| | application state can be one of the following: \
+| | ALL, NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING,
+| | FINISHED, FAILED, KILLED
*---------------+--------------+
-| -list | Lists all the Applications from RM |
+| -appTypes Types | Works with -list to filter applications based on input
+| | comma-separated list of application types.
*---------------+--------------+
-| -kill ApplicationId | Specify an application id |
+| -status ApplicationId | Prints the status of the application.
+*---------------+--------------+
+| -kill ApplicationId | Kills the application.
*---------------+--------------+
** node
@@ -81,9 +92,15 @@ Usage: yarn [--config confdir] COMMAND
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -status NodeId | Specify a node id |
+| -list | Lists all running nodes. Supports optional use of -states to filter
+| | nodes based on node state, and -all to list all nodes.
+*---------------+--------------+
+| -states States | Works with -list to filter nodes based on input
+| | comma-separated list of node states.
*---------------+--------------+
-| -list | Lists all the Nodes |
+| -all | Works with -list to list all nodes.
+*---------------+--------------+
+| -status NodeId | Prints the status report of the node.
*---------------+--------------+
** logs
@@ -91,19 +108,22 @@ Usage: yarn [--config confdir] COMMAND
Dump the container logs
-------
- Usage: yarn logs <options>
+ Usage: yarn logs -applicationId <application ID> <options>
-------
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -applicationId ApplicationId | Specify an application id |
+| -applicationId \<application ID\> | Specifies an application id |
*---------------+--------------+
-| -appOwner AppOwner | Specify an application owner |
+| -appOwner AppOwner | AppOwner (assumed to be current user if not
+| | specified)
*---------------+--------------+
-| -containerId ContainerId | Specify a container id |
+| -containerId ContainerId | ContainerId (must be specified if node address is
+| | specified)
*---------------+--------------+
-| -nodeAddress NodeAddress | Specify a node address |
+| -nodeAddress NodeAddress | NodeAddress in the format nodename:port (must be
+| | specified if container id is specified)
*---------------+--------------+
** classpath
@@ -158,7 +178,11 @@ Usage: yarn [--config confdir] COMMAND
-------
Usage: yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshUserToGroupsMapping]
[-refreshSuperUserGroupsConfiguration] [-refreshAdminAcls]
- [-refreshServiceAcl] [-help [cmd]]
+ [-refreshServiceAcl] [-getGroups [username]] [-help [cmd]]
+ [-transitionToActive <serviceId>]
+ [-transitionToStandby <serviceId>]
+ [-getServiceState <serviceId>]
+ [-checkHealth <serviceId>]
-------
*---------------+--------------+
@@ -176,8 +200,22 @@ Usage: yarn [--config confdir] COMMAND
*---------------+--------------+
| -refreshServiceAcl | Reload the service-level authorization policy file ResoureceManager will reload the authorization policy file. |
*---------------+--------------+
+| -getGroups [username] | Get groups the specified user belongs to.
+*---------------+--------------+
| -help [cmd] | Displays help for the given command or all commands if none is specified. |
*---------------+--------------+
+| -transitionToActive \<serviceId\> | Transitions the service into Active
+| | state.
+*---------------+--------------+
+| -transitionToStandby \<serviceId\> | Transitions the service into Standby
+| | state.
+*---------------+--------------+
+| -getServiceState \<serviceId\> | Returns the state of the service.
+*---------------+--------------+
+| -checkHealth \<serviceId\> | Requests that the service perform a health
+| | check. The RMAdmin tool will exit with a
+| | non-zero exit code if the check fails.
+*---------------+--------------+
** daemonlog
@@ -191,9 +229,9 @@ Usage: yarn [--config confdir] COMMAND
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -getlevel <host:port> <name> | Prints the log level of the daemon running at <host:port>. This command internally connects to http://<host:port>/logLevel?log=<name> |
+| -getlevel \<host:port\> \<name\> | Prints the log level of the daemon running at \<host:port\>. This command internally connects to http://\<host:port\>/logLevel?log=\<name\>
*---------------+--------------+
-| -setlevel <host:port> <name> <level> | Sets the log level of the daemon running at <host:port>. This command internally connects to http://<host:port>/logLevel?log=<name> |
+| -setlevel \<host:port\> \<name\> \<level\> | Sets the log level of the daemon running at \<host:port\>. This command internally connects to http://\<host:port\>/logLevel?log=\<name\>
*---------------+--------------+