You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/14 01:06:02 UTC
svn commit: r1467713 [3/3] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apach...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sat Apr 13 23:05:54 2013
@@ -73,13 +73,13 @@ public class ResourceTrackerService exte
private Server server;
private InetSocketAddress resourceTrackerAddress;
- private static final NodeHeartbeatResponse reboot = recordFactory
+ private static final NodeHeartbeatResponse resync = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
private static final NodeHeartbeatResponse shutDown = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
static {
- reboot.setNodeAction(NodeAction.REBOOT);
+ resync.setNodeAction(NodeAction.RESYNC);
shutDown.setNodeAction(NodeAction.SHUTDOWN);
}
@@ -220,7 +220,7 @@ public class ResourceTrackerService exte
if (rmNode == null) {
/* node does not exist */
LOG.info("Node not found rebooting " + remoteNodeStatus.getNodeId());
- return reboot;
+ return resync;
}
// Send ping
@@ -250,7 +250,7 @@ public class ResourceTrackerService exte
// TODO: Just sending reboot is not enough. Think more.
this.rmContext.getDispatcher().getEventHandler().handle(
new RMNodeEvent(nodeId, RMNodeEventType.REBOOTING));
- return reboot;
+ return resync;
}
// Heartbeat response
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Sat Apr 13 23:05:54 2013
@@ -77,6 +77,7 @@ public class AMLauncher implements Runna
RecordFactoryProvider.getRecordFactory(null);
private final AMLauncherEventType eventType;
private final RMContext rmContext;
+ private final Container masterContainer;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -88,34 +89,36 @@ public class AMLauncher implements Runna
this.eventType = eventType;
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
+ this.masterContainer = application.getMasterContainer();
}
private void connect() throws IOException {
- ContainerId masterContainerID = application.getMasterContainer().getId();
+ ContainerId masterContainerID = masterContainer.getId();
containerMgrProxy = getContainerMgrProxy(masterContainerID);
}
private void launch() throws IOException {
connect();
- ContainerId masterContainerID = application.getMasterContainer().getId();
+ ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext =
application.getSubmissionContext();
- LOG.info("Setting up container " + application.getMasterContainer()
+ LOG.info("Setting up container " + masterContainer
+ " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
createAMContainerLaunchContext(applicationContext, masterContainerID);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
+ request.setContainer(masterContainer);
containerMgrProxy.startContainer(request);
- LOG.info("Done launching container " + application.getMasterContainer()
+ LOG.info("Done launching container " + masterContainer
+ " for AM " + application.getAppAttemptId());
}
private void cleanup() throws IOException {
connect();
- ContainerId containerId = application.getMasterContainer().getId();
+ ContainerId containerId = masterContainer.getId();
StopContainerRequest stopRequest =
recordFactory.newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(containerId);
@@ -126,9 +129,7 @@ public class AMLauncher implements Runna
protected ContainerManager getContainerMgrProxy(
final ContainerId containerId) {
- Container container = application.getMasterContainer();
-
- final NodeId node = container.getNodeId();
+ final NodeId node = masterContainer.getNodeId();
final InetSocketAddress containerManagerBindAddress =
NetUtils.createSocketAddrForHost(node.getHost(), node.getPort());
@@ -138,8 +139,8 @@ public class AMLauncher implements Runna
.createRemoteUser(containerId.toString());
if (UserGroupInformation.isSecurityEnabled()) {
Token<ContainerTokenIdentifier> token =
- ProtoUtils.convertFromProtoFormat(container.getContainerToken(),
- containerManagerBindAddress);
+ ProtoUtils.convertFromProtoFormat(masterContainer
+ .getContainerToken(), containerManagerBindAddress);
currentUser.addToken(token);
}
return currentUser.doAs(new PrivilegedAction<ContainerManager>() {
@@ -165,30 +166,28 @@ public class AMLauncher implements Runna
new String[0])));
// Finalize the container
- container.setContainerId(containerID);
- container.setUser(applicationMasterContext.getUser());
- setupTokensAndEnv(container);
+ container.setUser(applicationMasterContext.getAMContainerSpec().getUser());
+ setupTokensAndEnv(container, containerID);
return container;
}
private void setupTokensAndEnv(
- ContainerLaunchContext container)
+ ContainerLaunchContext container, ContainerId containerID)
throws IOException {
Map<String, String> environment = container.getEnvironment();
-
environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
application.getWebProxyBase());
// Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be
// consumable by the AM.
- environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, container
- .getContainerId().toString());
- environment.put(ApplicationConstants.NM_HOST_ENV, application
- .getMasterContainer().getNodeId().getHost());
+ environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV,
+ containerID.toString());
+ environment.put(ApplicationConstants.NM_HOST_ENV, masterContainer
+ .getNodeId().getHost());
environment.put(ApplicationConstants.NM_PORT_ENV,
- String.valueOf(application.getMasterContainer().getNodeId().getPort()));
+ String.valueOf(masterContainer.getNodeId().getPort()));
String parts[] =
- application.getMasterContainer().getNodeHttpAddress().split(":");
+ masterContainer.getNodeHttpAddress().split(":");
environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]);
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Sat Apr 13 23:05:54 2013
@@ -690,7 +690,7 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(
new AppAddedSchedulerEvent(appAttempt.applicationAttemptId,
appAttempt.submissionContext.getQueue(),
- appAttempt.submissionContext.getUser()));
+ appAttempt.submissionContext.getAMContainerSpec().getUser()));
}
}
@@ -736,9 +736,10 @@ public class RMAppAttemptImpl implements
RMAppEventType.APP_ACCEPTED));
// Request a container for the AM.
- ResourceRequest request = BuilderUtils.newResourceRequest(
- AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt.submissionContext
- .getAMContainerSpec().getResource(), 1);
+ ResourceRequest request =
+ BuilderUtils.newResourceRequest(
+ AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt
+ .getSubmissionContext().getResource(), 1);
// SchedulerUtils.validateResourceRequests is not necessary because
// AM resource has been checked when submission
@@ -773,12 +774,8 @@ public class RMAppAttemptImpl implements
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
- // Updating CLC's resource is no longer necessary once YARN-486 is
- // completed, because nothing from Container to CLC will be copied into
- // CLC then.
- appAttempt.getSubmissionContext().getAMContainerSpec().setResource(
+ appAttempt.getSubmissionContext().setResource(
appAttempt.getMasterContainer().getResource());
-
RMStateStore store = appAttempt.rmContext.getStateStore();
appAttempt.storeAttempt(store);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Sat Apr 13 23:05:54 2013
@@ -462,7 +462,7 @@ public class FifoScheduler implements Re
FiCaSchedulerApp application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
+ application.getResourceRequest(priority, node.getHostName());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java Sat Apr 13 23:05:54 2013
@@ -59,8 +59,8 @@ public class AppAttemptInfo {
this.logsLink = join(HttpConfig.getSchemePrefix(),
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
- ConverterUtils.toString(masterContainer.getId()),
- "/", attempt.getSubmissionContext().getUser());
+ ConverterUtils.toString(masterContainer.getId()), "/",
+ attempt.getSubmissionContext().getAMContainerSpec().getUser());
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Sat Apr 13 23:05:54 2013
@@ -130,7 +130,7 @@ public class Application {
public synchronized void submit() throws IOException {
ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
context.setApplicationId(this.applicationId);
- context.setUser(this.user);
+ context.getAMContainerSpec().setUser(this.user);
context.setQueue(this.queue);
SubmitApplicationRequest request = recordFactory
.newRecordInstance(SubmitApplicationRequest.class);
@@ -340,7 +340,8 @@ public class Application {
// Launch the container
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(createCLC(container));
+ startRequest.setContainerLaunchContext(createCLC());
+ startRequest.setContainer(container);
nodeManager.startContainer(startRequest);
break;
}
@@ -396,11 +397,9 @@ public class Application {
}
}
- private ContainerLaunchContext createCLC(Container container) {
+ private ContainerLaunchContext createCLC() {
ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
- clc.setContainerId(container.getId());
clc.setUser(this.user);
- clc.setResource(container.getResource());
return clc;
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Sat Apr 13 23:05:54 2013
@@ -209,6 +209,7 @@ public class MockNodes {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String hostName = "host"+ nid;
+ final String nodeAddr = hostName + ":" + nid;
final int port = 123;
final NodeId nodeID = newNodeID(hostName, port);
final String httpAddress = httpAddr;
@@ -218,7 +219,7 @@ public class MockNodes {
nodeHealthStatus.setIsNodeHealthy(true);
nodeHealthStatus.setHealthReport("HealthyMe");
}
- return new MockRMNodeImpl(nodeID, hostName, httpAddress, perNode, rackName,
+ return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, rackName,
nodeHealthStatus, nid, hostName, state);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Sat Apr 13 23:05:54 2013
@@ -161,7 +161,6 @@ public class MockRM extends ResourceMana
.newRecord(ApplicationSubmissionContext.class);
sub.setApplicationId(appId);
sub.setApplicationName(name);
- sub.setUser(user);
sub.setMaxAppAttempts(maxAppAttempts);
if(unmanaged) {
sub.setUnmanagedAM(true);
@@ -171,13 +170,13 @@ public class MockRM extends ResourceMana
}
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
- Resource capability = Records.newRecord(Resource.class);
+ final Resource capability = Records.newRecord(Resource.class);
capability.setMemory(masterMemory);
- clc.setResource(capability);
+ sub.setResource(capability);
clc.setApplicationACLs(acls);
+ clc.setUser(user);
sub.setAMContainerSpec(clc);
req.setApplicationSubmissionContext(sub);
-
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
PrivilegedAction<SubmitApplicationResponse> action =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Sat Apr 13 23:05:54 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
@@ -162,11 +161,10 @@ public class NodeManager implements Cont
synchronized public StartContainerResponse startContainer(
StartContainerRequest request)
throws YarnRemoteException {
- ContainerLaunchContext containerLaunchContext =
- request.getContainerLaunchContext();
-
+ Container requestContainer = request.getContainer();
+
ApplicationId applicationId =
- containerLaunchContext.getContainerId().getApplicationAttemptId().
+ requestContainer.getId().getApplicationAttemptId().
getApplicationId();
List<Container> applicationContainers = containers.get(applicationId);
@@ -177,18 +175,18 @@ public class NodeManager implements Cont
// Sanity check
for (Container container : applicationContainers) {
- if (container.getId().compareTo(containerLaunchContext.getContainerId())
+ if (container.getId().compareTo(requestContainer.getId())
== 0) {
throw new IllegalStateException(
- "Container " + containerLaunchContext.getContainerId() +
+ "Container " + requestContainer.getId() +
" already setup on node " + containerManagerAddress);
}
}
Container container =
- BuilderUtils.newContainer(containerLaunchContext.getContainerId(),
+ BuilderUtils.newContainer(requestContainer.getId(),
this.nodeId, nodeHttpAddress,
- containerLaunchContext.getResource(),
+ requestContainer.getResource(),
null, null // DKDC - Doesn't matter
);
@@ -197,8 +195,8 @@ public class NodeManager implements Cont
"", -1000);
applicationContainers.add(container);
containerStatusMap.put(container, containerStatus);
- Resources.subtractFrom(available, containerLaunchContext.getResource());
- Resources.addTo(used, containerLaunchContext.getResource());
+ Resources.subtractFrom(available, requestContainer.getResource());
+ Resources.addTo(used, requestContainer.getResource());
if(LOG.isDebugEnabled()) {
LOG.debug("startContainer:" + " node=" + containerManagerAddress
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Sat Apr 13 23:05:54 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.Service;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -503,6 +504,10 @@ public class TestAppManager{
RMApp appOrig = rmContext.getRMApps().get(appID);
Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());
+ ContainerLaunchContext clc =
+ BuilderUtils.newContainerLaunchContext(null, null, null, null, null,
+ null, null);
+ context.setAMContainerSpec(clc);
// our testApp1 should be rejected and original app with same id should be left in place
appMonitor.submitApplication(context);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Sat Apr 13 23:05:54 2013
@@ -169,7 +169,7 @@ public class TestApplicationACLs {
ContainerLaunchContext amContainer = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
Resource resource = BuilderUtils.newResource(1024, 1);
- amContainer.setResource(resource);
+ context.setResource(resource);
amContainer.setApplicationACLs(acls);
context.setAMContainerSpec(amContainer);
submitRequest.setApplicationSubmissionContext(context);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Sat Apr 13 23:05:54 2013
@@ -134,7 +134,7 @@ public class TestApplicationMasterLaunch
Assert.assertEquals(app.getSubmitTime(),
containerManager.submitTimeAtContainerManager);
Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
- .getSubmissionContext().getAMContainerSpec().getContainerId()
+ .getMasterContainer().getId()
.toString(), containerManager.containerIdAtContainerManager);
Assert.assertEquals(nm1.getNodeId().getHost(),
containerManager.nmHostAtContainerManager);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Sat Apr 13 23:05:54 2013
@@ -340,7 +340,7 @@ public class TestClientRMService {
final SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId);
Resource resource = Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1);
- when(submitRequest.getApplicationSubmissionContext().getAMContainerSpec()
+ when(submitRequest.getApplicationSubmissionContext()
.getResource()).thenReturn(resource);
final ClientRMService rmService =
@@ -364,16 +364,17 @@ public class TestClientRMService {
String queue = MockApps.newQueue();
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
+
Resource resource = Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- when(amContainerSpec.getResource()).thenReturn(resource);
ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
- when(submissionContext.getUser()).thenReturn(user);
- when(submissionContext.getQueue()).thenReturn(queue);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
+ when(submissionContext.getAMContainerSpec().getUser()).thenReturn(user);
+ when(submissionContext.getQueue()).thenReturn(queue);
when(submissionContext.getApplicationId()).thenReturn(appId);
-
+ when(submissionContext.getResource()).thenReturn(resource);
+
SubmitApplicationRequest submitRequest =
recordFactory.newRecordInstance(SubmitApplicationRequest.class);
submitRequest.setApplicationSubmissionContext(submissionContext);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Sat Apr 13 23:05:54 2013
@@ -225,9 +225,9 @@ public class TestRMRestart {
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
- Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
+ Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
hbResponse = nm2.nodeHeartbeat(true);
- Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
+ Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
// new NM to represent NM re-register
nm1 = rm2.registerNode("h1:1234", 15120);
@@ -235,9 +235,9 @@ public class TestRMRestart {
// verify no more reboot response sent
hbResponse = nm1.nodeHeartbeat(true);
- Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
+ Assert.assertTrue(NodeAction.RESYNC != hbResponse.getNodeAction());
hbResponse = nm2.nodeHeartbeat(true);
- Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
+ Assert.assertTrue(NodeAction.RESYNC != hbResponse.getNodeAction());
// assert app1 attempt is saved
attempt1 = loadedApp1.getCurrentAppAttempt();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Sat Apr 13 23:05:54 2013
@@ -282,7 +282,7 @@ public class TestResourceTrackerService
nodeHeartbeat = nm2.nodeHeartbeat(
new HashMap<ApplicationId, List<ContainerStatus>>(), true, -100);
- Assert.assertTrue(NodeAction.REBOOT.equals(nodeHeartbeat.getNodeAction()));
+ Assert.assertTrue(NodeAction.RESYNC.equals(nodeHeartbeat.getNodeAction()));
checkRebootedNMCount(rm, ++initialMetricCount);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java Sat Apr 13 23:05:54 2013
@@ -130,6 +130,6 @@ public class TestRMNMRPCResponseId {
nodeStatus.setResponseId(0);
response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
- Assert.assertTrue(NodeAction.REBOOT.equals(response.getNodeAction()));
+ Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Sat Apr 13 23:05:54 2013
@@ -200,14 +200,14 @@ public class TestRMAppAttemptTransitions
final String user = MockApps.newUserName();
final String queue = MockApps.newQueue();
submissionContext = mock(ApplicationSubmissionContext.class);
- when(submissionContext.getUser()).thenReturn(user);
when(submissionContext.getQueue()).thenReturn(queue);
Resource resource = BuilderUtils.newResource(1536, 1);
ContainerLaunchContext amContainerSpec =
- BuilderUtils.newContainerLaunchContext(null, user, resource, null, null,
+ BuilderUtils.newContainerLaunchContext(user, null, null,
null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
-
+ when(submissionContext.getResource()).thenReturn(resource);
+
unmanagedAM = false;
application = mock(RMApp.class);
@@ -494,9 +494,6 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(
new RMAppAttemptStoredEvent(
applicationAttempt.getAppAttemptId(), null));
- assertEquals(resource,
- applicationAttempt.getSubmissionContext()
- .getAMContainerSpec().getResource());
testAppAttemptAllocatedState(container);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Apr 13 23:05:54 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -1406,6 +1408,10 @@ public class TestFairScheduler {
ApplicationMasterService masterService =
new ApplicationMasterService(resourceManager.getRMContext(), scheduler);
ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl();
+ ContainerLaunchContext clc =
+ BuilderUtils.newContainerLaunchContext(user, null, null, null, null,
+ null, null);
+ submissionContext.setAMContainerSpec(clc);
RMApp application =
new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user,
queue, submissionContext, scheduler, masterService,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Sat Apr 13 23:05:54 2013
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import junit.framework.Assert;
@@ -28,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -35,15 +38,22 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
@@ -55,6 +65,9 @@ public class TestFifoScheduler {
private ResourceManager resourceManager = null;
+ private static final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
@Before
public void setUp() throws Exception {
resourceManager = new ResourceManager();
@@ -78,14 +91,38 @@ public class TestFifoScheduler {
.getRMContext());
}
- @Test
+ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
+ ApplicationAttemptId attId = recordFactory
+ .newRecordInstance(ApplicationAttemptId.class);
+ ApplicationId appIdImpl = recordFactory
+ .newRecordInstance(ApplicationId.class);
+ appIdImpl.setId(appId);
+ attId.setAttemptId(attemptId);
+ attId.setApplicationId(appIdImpl);
+ return attId;
+ }
+
+ private ResourceRequest createResourceRequest(int memory, String host,
+ int priority, int numContainers) {
+ ResourceRequest request = recordFactory
+ .newRecordInstance(ResourceRequest.class);
+ request.setCapability(Resources.createResource(memory));
+ request.setHostName(host);
+ request.setNumContainers(numContainers);
+ Priority prio = recordFactory.newRecordInstance(Priority.class);
+ prio.setPriority(priority);
+ request.setPriority(prio);
+ return request;
+ }
+
+ @Test(timeout=5000)
public void testFifoSchedulerCapacityWhenNoNMs() {
FifoScheduler scheduler = new FifoScheduler();
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity());
}
- @Test
+ @Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
RMContext rmContext = new RMContextImpl(dispatcher, null,
@@ -111,6 +148,59 @@ public class TestFifoScheduler {
Assert.assertEquals(1, metrics.getAppsSubmitted());
}
+ @Test(timeout=2000)
+ public void testNodeLocalAssignment() throws Exception {
+ AsyncDispatcher dispatcher = new InlineDispatcher();
+ RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
+ null, null, null);
+
+ FifoScheduler scheduler = new FifoScheduler();
+ scheduler.reinitialize(new Configuration(), rmContext);
+
+ RMNode node0 = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024 * 64), 1234);
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
+ scheduler.handle(nodeEvent1);
+
+ int _appId = 1;
+ int _appAttemptId = 1;
+ ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
+ _appAttemptId);
+ AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId,
+ "queue1", "user1");
+ scheduler.handle(appEvent1);
+
+ int memory = 64;
+ int nConts = 3;
+ int priority = 20;
+
+ List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+ ResourceRequest nodeLocal = createResourceRequest(memory,
+ node0.getHostName(), priority, nConts);
+ ResourceRequest rackLocal = createResourceRequest(memory,
+ node0.getRackName(), priority, nConts);
+ ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
+ nConts);
+ ask.add(nodeLocal);
+ ask.add(rackLocal);
+ ask.add(any);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>());
+
+ NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
+
+ // Before the node update event, there are 3 local requests outstanding
+ Assert.assertEquals(3, nodeLocal.getNumContainers());
+
+ scheduler.handle(node0Update);
+
+ // After the node update event, check that there are no more local requests
+ // outstanding
+ Assert.assertEquals(0, nodeLocal.getNumContainers());
+ //Also check that the containers were scheduled
+ SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
+ Assert.assertEquals(3, info.getLiveContainers().size());
+ }
+
// @Test
public void testFifoScheduler() throws Exception {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Sat Apr 13 23:05:54 2013
@@ -1079,8 +1079,9 @@ public class TestRMWebServicesApps exten
.getMasterContainer().getNodeId().toString(), nodeId);
assertTrue("logsLink doesn't match",
logsLink.startsWith("http://"));
- assertTrue("logsLink doesn't contain user info",
- logsLink.endsWith("/" + appAttempt.getSubmissionContext().getUser()));
+ assertTrue(
+ "logsLink doesn't contain user info", logsLink.endsWith("/"
+ + appAttempt.getSubmissionContext().getAMContainerSpec().getUser()));
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Sat Apr 13 23:05:54 2013
@@ -357,8 +357,13 @@ public class TestContainerManagerSecurit
LOG.info("Going to contact NM with expired token");
ContainerLaunchContext context = createContainerLaunchContextForTest(newTokenId);
+ Container container =
+ BuilderUtils.newContainer(newTokenId.getContainerID(), null, null,
+ BuilderUtils.newResource(newTokenId.getResource().getMemory(),
+ newTokenId.getResource().getVirtualCores()), null, null);
StartContainerRequest request = Records.newRecord(StartContainerRequest.class);
request.setContainerLaunchContext(context);
+ request.setContainer(container);
//Calling startContainer with an expired token.
try {
@@ -402,18 +407,19 @@ public class TestContainerManagerSecurit
Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") :
Arrays.asList("sleep", "100");
- ContainerLaunchContext amContainer = BuilderUtils
- .newContainerLaunchContext(null, "testUser", BuilderUtils
- .newResource(1024, 1), Collections.<String, LocalResource>emptyMap(),
- new HashMap<String, String>(), cmd,
- new HashMap<String, ByteBuffer>(), null,
- new HashMap<ApplicationAccessType, String>());
+ ContainerLaunchContext amContainer =
+ BuilderUtils.newContainerLaunchContext("testUser",
+ Collections.<String, LocalResource> emptyMap(),
+ new HashMap<String, String>(), cmd,
+ new HashMap<String, ByteBuffer>(), null,
+ new HashMap<ApplicationAccessType, String>());
ApplicationSubmissionContext appSubmissionContext = recordFactory
.newRecordInstance(ApplicationSubmissionContext.class);
appSubmissionContext.setApplicationId(appID);
- appSubmissionContext.setUser("testUser");
appSubmissionContext.setAMContainerSpec(amContainer);
+ appSubmissionContext.getAMContainerSpec().setUser("testUser");
+ appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1));
SubmitApplicationRequest submitRequest = recordFactory
.newRecordInstance(SubmitApplicationRequest.class);
@@ -539,8 +545,11 @@ public class TestContainerManagerSecurit
// Authenticated but unauthorized, due to wrong resource
ContainerLaunchContext context =
createContainerLaunchContextForTest(tokenId);
- context.getResource().setMemory(2048); // Set a different resource size.
+ Container container =
+ BuilderUtils.newContainer(tokenId.getContainerID(), null, null,
+ BuilderUtils.newResource(2048, 1), null, null);
request.setContainerLaunchContext(context);
+ request.setContainer(container);
try {
client.startContainer(request);
fail("Connection initiation with unauthorized "
@@ -551,7 +560,7 @@ public class TestContainerManagerSecurit
"Unauthorized request to start container. "));
Assert.assertTrue(e.getMessage().contains(
"\nExpected resource " + tokenId.getResource().toString()
- + " but found " + context.getResource().toString()));
+ + " but found " + container.getResource().toString()));
}
}
@@ -563,7 +572,12 @@ public class TestContainerManagerSecurit
ContainerLaunchContext context =
createContainerLaunchContextForTest(tokenId);
context.setUser("Saruman"); // Set a different user-name.
+ Container container =
+ BuilderUtils.newContainer(tokenId.getContainerID(), null, null,
+ BuilderUtils.newResource(tokenId.getResource().getMemory(), tokenId
+ .getResource().getVirtualCores()), null, null);
request.setContainerLaunchContext(context);
+ request.setContainer(container);
try {
client.startContainer(request);
fail("Connection initiation with unauthorized "
@@ -581,12 +595,8 @@ public class TestContainerManagerSecurit
private ContainerLaunchContext createContainerLaunchContextForTest(
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =
- BuilderUtils.newContainerLaunchContext(tokenId.getContainerID(),
- "testUser",
- BuilderUtils.newResource(
- tokenId.getResource().getMemory(),
- tokenId.getResource().getVirtualCores()),
- new HashMap<String, LocalResource>(),
+ BuilderUtils.newContainerLaunchContext(
+ "testUser", new HashMap<String, LocalResource>(),
new HashMap<String, String>(), new ArrayList<String>(),
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java Sat Apr 13 23:05:54 2013
@@ -133,10 +133,10 @@ public class TestDiskFailures {
dirSvc.init(conf);
List<String> localDirs = dirSvc.getLocalDirs();
Assert.assertEquals(1, localDirs.size());
- Assert.assertEquals(localDir2, localDirs.get(0));
+ Assert.assertEquals(new Path(localDir2).toString(), localDirs.get(0));
List<String> logDirs = dirSvc.getLogDirs();
Assert.assertEquals(1, logDirs.size());
- Assert.assertEquals(logDir1, logDirs.get(0));
+ Assert.assertEquals(new Path(logDir1).toString(), logDirs.get(0));
}
private void testDirsFailures(boolean localORLogDirs) throws IOException {