You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/07/24 05:42:54 UTC
svn commit: r1506392 [2/3] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Wed Jul 24 03:42:52 2013
@@ -23,12 +23,12 @@ import static org.apache.hadoop.service.
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
@@ -50,25 +50,25 @@ import org.apache.hadoop.service.Service
import org.apache.hadoop.service.ServiceStateChangeListener;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.InvalidContainerException;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -130,8 +130,6 @@ public class ContainerManagerImpl extend
private final NodeStatusUpdater nodeStatusUpdater;
- private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
protected LocalDirsHandlerService dirsHandler;
protected final AsyncDispatcher dispatcher;
private final ApplicationACLsManager aclsManager;
@@ -316,18 +314,25 @@ public class ContainerManagerImpl extend
return resultId;
}
+ protected void authorizeUser(UserGroupInformation remoteUgi,
+ NMTokenIdentifier nmTokenIdentifier) throws YarnException {
+ if (!remoteUgi.getUserName().equals(
+ nmTokenIdentifier.getApplicationAttemptId().toString())) {
+ throw RPCUtil.getRemoteException("Expected applicationAttemptId: "
+ + remoteUgi.getUserName() + "Found: "
+ + nmTokenIdentifier.getApplicationAttemptId());
+ }
+ }
+
/**
* @param containerTokenIdentifier
* of the container to be started
- * @param ugi
- * ugi corresponding to the remote end making the api-call
* @throws YarnException
*/
@Private
@VisibleForTesting
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier,
- UserGroupInformation ugi) throws YarnException {
+ ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
ContainerId containerId = containerTokenIdentifier.getContainerID();
String containerIDStr = containerId.toString();
@@ -342,12 +347,6 @@ public class ContainerManagerImpl extend
.append(" was used for starting container with container token")
.append(" issued for application attempt : ")
.append(containerId.getApplicationAttemptId());
- } else if (!ugi.getUserName().equals(
- nmTokenIdentifier.getApplicationAttemptId().toString())) {
- unauthorized = true;
- messageBuilder.append("\nExpected applicationAttemptId: ")
- .append(ugi.getUserName()).append(" Found: ")
- .append(nmTokenIdentifier.getApplicationAttemptId().toString());
} else if (!this.context.getContainerTokenSecretManager()
.isValidStartContainerRequest(containerTokenIdentifier)) {
// Is the container being relaunched? Or RPC layer let startCall with
@@ -363,7 +362,6 @@ public class ContainerManagerImpl extend
.append(System.currentTimeMillis()).append(" found ")
.append(containerTokenIdentifier.getExpiryTimeStamp());
}
-
if (unauthorized) {
String msg = messageBuilder.toString();
LOG.error(msg);
@@ -372,18 +370,53 @@ public class ContainerManagerImpl extend
}
/**
- * Start a container on this NodeManager.
+ * Start a list of containers on this NodeManager.
*/
- @SuppressWarnings("unchecked")
@Override
- public StartContainerResponse startContainer(StartContainerRequest request)
- throws YarnException, IOException {
-
+ public StartContainersResponse
+ startContainers(StartContainersRequest requests) throws YarnException,
+ IOException {
if (blockNewContainerRequests.get()) {
throw new NMNotYetReadyException(
"Rejecting new containers as NodeManager has not"
+ " yet connected with ResourceManager");
}
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
+ authorizeUser(remoteUgi,nmTokenIdentifier);
+ List<ContainerId> succeededContainers = new ArrayList<ContainerId>();
+ Map<ContainerId, SerializedException> failedContainers =
+ new HashMap<ContainerId, SerializedException>();
+ for (StartContainerRequest request : requests.getStartContainerRequests()) {
+ ContainerId containerId = null;
+ try {
+ ContainerTokenIdentifier containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(request.getContainerToken());
+ verifyAndGetContainerTokenIdentifier(request.getContainerToken(),
+ containerTokenIdentifier);
+ containerId = containerTokenIdentifier.getContainerID();
+ startContainerInternal(nmTokenIdentifier, containerTokenIdentifier,
+ request);
+ succeededContainers.add(containerId);
+ } catch (YarnException e) {
+ failedContainers.put(containerId, SerializedException.newInstance(e));
+ } catch (InvalidToken ie) {
+ failedContainers.put(containerId, SerializedException.newInstance(ie));
+ throw ie;
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+ }
+
+ return StartContainersResponse.newInstance(auxiliaryServices.getMetaData(),
+ succeededContainers, failedContainers);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void startContainerInternal(NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ StartContainerRequest request) throws YarnException, IOException {
+
/*
* 1) It should save the NMToken into NMTokenSecretManager. This is done
* here instead of RPC layer because at the time of opening/authenticating
@@ -395,18 +428,8 @@ public class ContainerManagerImpl extend
* belongs to correct Node Manager (part of retrieve password). c) It has
* correct RMIdentifier. d) It is not expired.
*/
- // update NMToken
-
- UserGroupInformation remoteUgi = getRemoteUgi();
- NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
-
- // Validate containerToken
- ContainerTokenIdentifier containerTokenIdentifier =
- verifyAndGetContainerTokenIdentifier(request.getContainerToken());
-
- authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier,
- remoteUgi);
-
+ authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier);
+
if (containerTokenIdentifier.getRMIdentifer() != nodeStatusUpdater
.getRMIdentifier()) {
// Is the container coming from unknown RM
@@ -415,9 +438,9 @@ public class ContainerManagerImpl extend
.append(" rejected as it is allocated by a previous RM");
throw new InvalidContainerException(sb.toString());
}
-
+ // update NMToken
updateNMTokenIdentifier(nmTokenIdentifier);
-
+
ContainerId containerId = containerTokenIdentifier.getContainerID();
String containerIdStr = containerId.toString();
String user = containerTokenIdentifier.getApplicationSubmitter();
@@ -461,26 +484,16 @@ public class ContainerManagerImpl extend
containerTokenIdentifier);
NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER,
"ContainerManageImpl", applicationID, containerId);
- StartContainerResponse response =
- recordFactory.newRecordInstance(StartContainerResponse.class);
- response.setAllServicesMetaData(auxiliaryServices.getMetaData());
// TODO launchedContainer misplaced -> doesn't necessarily mean a container
// launch. A finished Application will not launch containers.
metrics.launchedContainer();
- metrics.allocateContainer(containerTokenIdentifier.getResource());
- return response;
+ metrics.allocateContainer(containerTokenIdentifier.getResource());
}
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
- org.apache.hadoop.yarn.api.records.Token token) throws YarnException,
+ org.apache.hadoop.yarn.api.records.Token token,
+ ContainerTokenIdentifier containerTokenIdentifier) throws YarnException,
InvalidToken {
- ContainerTokenIdentifier containerTokenIdentifier = null;
- try {
- containerTokenIdentifier =
- BuilderUtils.newContainerTokenIdentifier(token);
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
- }
byte[] password =
context.getContainerTokenSecretManager().retrievePassword(
containerTokenIdentifier);
@@ -528,21 +541,37 @@ public class ContainerManagerImpl extend
}
/**
- * Stop the container running on this NodeManager.
+ * Stop a list of containers running on this NodeManager.
*/
@Override
- @SuppressWarnings("unchecked")
- public StopContainerResponse stopContainer(StopContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest requests)
throws YarnException, IOException {
- ContainerId containerID = request.getContainerId();
+ List<ContainerId> succeededRequests = new ArrayList<ContainerId>();
+ Map<ContainerId, SerializedException> failedRequests =
+ new HashMap<ContainerId, SerializedException>();
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
+ for (ContainerId id : requests.getContainerIds()) {
+ try {
+ stopContainerInternal(identifier, id);
+ succeededRequests.add(id);
+ } catch (YarnException e) {
+ failedRequests.put(id, SerializedException.newInstance(e));
+ }
+ }
+ return StopContainersResponse
+ .newInstance(succeededRequests, failedRequests);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier,
+ ContainerId containerID) throws YarnException {
String containerIDStr = containerID.toString();
Container container = this.context.getContainers().get(containerID);
- LOG.info("Getting container-status for " + containerIDStr);
- authorizeGetAndStopContainerRequest(containerID, container, true);
-
- StopContainerResponse response =
- recordFactory.newRecordInstance(StopContainerResponse.class);
+ LOG.info("Stopping container with container Id: " + containerIDStr);
+ authorizeGetAndStopContainerRequest(containerID, container, true,
+ nmTokenIdentifier);
dispatcher.getEventHandler().handle(
new ContainerKillEvent(containerID,
@@ -555,37 +584,51 @@ public class ContainerManagerImpl extend
// TODO: Move this code to appropriate place once kill_container is
// implemented.
nodeStatusUpdater.sendOutofBandHeartBeat();
-
- return response;
}
+ /**
+ * Get a list of container statuses running on this NodeManager
+ */
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException, IOException {
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException, IOException {
+
+ List<ContainerStatus> succeededRequests = new ArrayList<ContainerStatus>();
+ Map<ContainerId, SerializedException> failedRequests =
+ new HashMap<ContainerId, SerializedException>();
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
+ for (ContainerId id : request.getContainerIds()) {
+ try {
+ ContainerStatus status = getContainerStatusInternal(id, identifier);
+ succeededRequests.add(status);
+ } catch (YarnException e) {
+ failedRequests.put(id, SerializedException.newInstance(e));
+ }
+ }
+ return GetContainerStatusesResponse.newInstance(succeededRequests,
+ failedRequests);
+ }
- ContainerId containerID = request.getContainerId();
+ private ContainerStatus getContainerStatusInternal(ContainerId containerID,
+ NMTokenIdentifier nmTokenIdentifier) throws YarnException {
String containerIDStr = containerID.toString();
Container container = this.context.getContainers().get(containerID);
LOG.info("Getting container-status for " + containerIDStr);
- authorizeGetAndStopContainerRequest(containerID, container, false);
+ authorizeGetAndStopContainerRequest(containerID, container, false,
+ nmTokenIdentifier);
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
- GetContainerStatusResponse response =
- recordFactory.newRecordInstance(GetContainerStatusResponse.class);
- response.setStatus(containerStatus);
- return response;
+ return containerStatus;
}
@Private
@VisibleForTesting
protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
- Container container, boolean stopRequest) throws YarnException {
-
- UserGroupInformation remoteUgi = getRemoteUgi();
- NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
-
+ Container container, boolean stopRequest, NMTokenIdentifier identifier)
+ throws YarnException {
/*
* For get/stop container status; we need to verify that 1) User (NMToken)
* application attempt only has started container. 2) Requested containerId
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Wed Jul 24 03:42:52 2013
@@ -103,7 +103,7 @@ public class ContainerImpl implements Co
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext, Credentials creds,
NodeManagerMetrics metrics,
- ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
+ ContainerTokenIdentifier containerTokenIdentifier) {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Wed Jul 24 03:42:52 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -288,11 +289,10 @@ public class ContainerLocalizer {
stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
} catch (ExecutionException e) {
stat.setStatus(ResourceStatusType.FETCH_FAILURE);
- stat.setException(
- YarnServerBuilderUtils.newSerializedException(e.getCause()));
+ stat.setException(SerializedException.newInstance(e.getCause()));
} catch (CancellationException e) {
stat.setStatus(ResourceStatusType.FETCH_FAILURE);
- stat.setException(YarnServerBuilderUtils.newSerializedException(e));
+ stat.setException(SerializedException.newInstance(e));
}
// TODO shouldn't remove until ACK
i.remove();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Wed Jul 24 03:42:52 2013
@@ -195,14 +195,13 @@ public class DummyContainerManager exten
@Override
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier,
- UserGroupInformation ugi) throws YarnException {
+ ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
// do nothing
}
@Override
protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
- Container container, boolean stopRequest) throws YarnException {
+ Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException {
// do nothing
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java Wed Jul 24 03:42:52 2013
@@ -146,7 +146,7 @@ public class TestContainerManagerWithLCE
@Override
public void testContainerLaunchFromPreviousRM() throws InterruptedException,
- IOException {
+ IOException, YarnException {
// Don't run the test if the binary is not available.
if (!shouldRunTest()) {
LOG.info("LCE binary path is not passed. Not running the test");
@@ -155,6 +155,29 @@ public class TestContainerManagerWithLCE
LOG.info("Running testContainerLaunchFromPreviousRM");
super.testContainerLaunchFromPreviousRM();
}
+
+ @Override
+ public void testMultipleContainersLaunch() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testContainerLaunchFromPreviousRM");
+ super.testMultipleContainersLaunch();
+ }
+
+ @Override
+ public void testMultipleContainersStopAndGetStatus() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testContainerLaunchFromPreviousRM");
+ super.testMultipleContainersStopAndGetStatus();
+ }
+
private boolean shouldRunTest() {
return System
.getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Wed Jul 24 03:42:52 2013
@@ -20,18 +20,19 @@ package org.apache.hadoop.yarn.server.no
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Test;
@@ -62,7 +62,7 @@ public class TestEventFlow {
private static File remoteLogDir = new File("target",
TestEventFlow.class.getName() + "-remoteLogDir").getAbsoluteFile();
private static final long SIMULATED_RM_IDENTIFIER = 1234;
-
+
@Test
public void testSuccessfulContainerLaunch() throws InterruptedException,
IOException, YarnException {
@@ -140,21 +140,25 @@ public class TestEventFlow {
ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0);
String user = "testing";
- StartContainerRequest request =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- request.setContainerLaunchContext(launchContext);
- request.setContainerToken(TestContainerManager.createContainerToken(cID,
- SIMULATED_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
- containerManager.startContainer(request);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(launchContext,
+ TestContainerManager.createContainerToken(cID,
+ SIMULATED_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
ContainerState.RUNNING);
- StopContainerRequest stopRequest =
- recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(cID);
- containerManager.stopContainer(stopRequest);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cID);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ containerManager.stopContainers(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
ContainerState.COMPLETE);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java Wed Jul 24 03:42:52 2013
@@ -39,8 +39,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -129,13 +130,17 @@ public class TestNodeManagerReboot {
List<String> commands = new ArrayList<String>();
containerLaunchContext.setCommands(commands);
- final StartContainerRequest startRequest =
- Records.newRecord(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
NodeId nodeId = nm.getNMContext().getNodeId();
- startRequest.setContainerToken(TestContainerManager.createContainerToken(
- cId, 0, nodeId, destinationFile, nm.getNMContext()
- .getContainerTokenSecretManager()));
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ TestContainerManager.createContainerToken(
+ cId, 0, nodeId, destinationFile, nm.getNMContext()
+ .getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ final StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+
final UserGroupInformation currentUser =
UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId()
.toString());
@@ -145,16 +150,17 @@ public class TestNodeManagerReboot {
currentUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws YarnException, IOException {
- nm.getContainerManager().startContainer(startRequest);
+ nm.getContainerManager().startContainers(allRequests);
return null;
}
});
- GetContainerStatusRequest request =
- Records.newRecord(GetContainerStatusRequest.class);
- request.setContainerId(cId);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(containerIds);
Container container =
- nm.getNMContext().getContainers().get(request.getContainerId());
+ nm.getNMContext().getContainers().get(request.getContainerIds().get(0));
final int MAX_TRIES = 20;
int numTries = 0;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java Wed Jul 24 03:42:52 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.no
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CyclicBarrier;
@@ -31,6 +33,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -282,15 +285,18 @@ public class TestNodeManagerResync {
try {
while (!isStopped && numContainers < 10) {
ContainerId cId = TestNodeManagerShutdown.createContainerId();
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(null);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ null);
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
System.out.println("no. of containers to be launched: "
+ numContainers);
numContainers++;
try {
- getContainerManager().startContainer(startRequest);
+ getContainerManager().startContainers(allRequests);
} catch (YarnException e) {
numContainersRejected++;
Assert.assertTrue(e.getMessage().contains(
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Wed Jul 24 03:42:52 2013
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -40,8 +41,9 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -53,7 +55,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -181,12 +182,6 @@ public class TestNodeManagerShutdown {
containerLaunchContext.setLocalResources(localResources);
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest
- .setContainerToken(TestContainerManager.createContainerToken(cId, 0,
- nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
final InetSocketAddress containerManagerBindAddress =
NetUtils.createSocketAddrForHost("127.0.0.1", 12345);
UserGroupInformation currentUser = UserGroupInformation
@@ -210,13 +205,22 @@ public class TestNodeManagerShutdown {
containerManagerBindAddress, conf);
}
});
- containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ TestContainerManager.createContainerToken(cId, 0,
+ nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
- GetContainerStatusRequest request =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- request.setContainerId(cId);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
- containerManager.getContainerStatus(request).getStatus();
+ containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
Assert.assertEquals(ContainerState.RUNNING, containerStatus.getState());
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Wed Jul 24 03:42:52 2013
@@ -45,8 +45,8 @@ import org.apache.hadoop.io.retry.RetryP
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.ServiceOperations;
import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.service.ServiceOperations;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
-import org.apache.hadoop.yarn.server.api.ServerRMProxy;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java Wed Jul 24 03:42:52 2013
@@ -21,16 +21,21 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import junit.framework.Assert;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto;
@@ -42,7 +47,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
-import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
@@ -80,7 +84,7 @@ public class TestPBRecordImpl {
e.setStackTrace(new StackTraceElement[] {
new StackTraceElement("foo", "bar", "baz", 10),
new StackTraceElement("sbb", "one", "onm", 10) });
- ret.setException(YarnServerBuilderUtils.newSerializedException(e));
+ ret.setException(SerializedException.newInstance(e));
return ret;
}
@@ -176,4 +180,33 @@ public class TestPBRecordImpl {
assertEquals(createResource(), rsrcD.getResourceSpecs().get(0).getResource());
}
+
+ @Test(timeout=10000)
+ public void testSerializedExceptionDeSer() throws Exception{
+ // without cause
+ YarnException yarnEx = new YarnException("Yarn_Exception");
+ SerializedException serEx = SerializedException.newInstance(yarnEx);
+ Throwable throwable = serEx.deSerialize();
+ Assert.assertEquals(yarnEx.getClass(), throwable.getClass());
+ Assert.assertEquals(yarnEx.getMessage(), throwable.getMessage());
+
+ // with cause
+ IOException ioe = new IOException("Test_IOException");
+ RuntimeException runtimeException =
+ new RuntimeException("Test_RuntimeException", ioe);
+ YarnException yarnEx2 =
+ new YarnException("Test_YarnException", runtimeException);
+
+ SerializedException serEx2 = SerializedException.newInstance(yarnEx2);
+ Throwable throwable2 = serEx2.deSerialize();
+ throwable2.printStackTrace();
+ Assert.assertEquals(yarnEx2.getClass(), throwable2.getClass());
+ Assert.assertEquals(yarnEx2.getMessage(), throwable2.getMessage());
+
+ Assert.assertEquals(runtimeException.getClass(), throwable2.getCause().getClass());
+ Assert.assertEquals(runtimeException.getMessage(), throwable2.getCause().getMessage());
+
+ Assert.assertEquals(ioe.getClass(), throwable2.getCause().getCause().getClass());
+ Assert.assertEquals(ioe.getMessage(), throwable2.getCause().getCause().getMessage());
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Wed Jul 24 03:42:52 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.no
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import junit.framework.Assert;
@@ -32,7 +34,7 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -189,15 +191,18 @@ public abstract class BaseContainerManag
@Override
protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
- Container container, boolean stopRequest) throws YarnException {
+ Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException {
// do nothing
}
-
+ @Override
+ protected void authorizeUser(UserGroupInformation remoteUgi,
+ NMTokenIdentifier nmTokenIdentifier) {
+ // do nothing
+ }
@Override
protected void authorizeStartRequest(
NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier,
- UserGroupInformation ugi) throws YarnException {
+ ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
// do nothing
}
@@ -238,18 +243,20 @@ public abstract class BaseContainerManag
public static void waitForContainerState(ContainerManagementProtocol containerManager,
ContainerId containerID, ContainerState finalState, int timeOutMax)
throws InterruptedException, YarnException, IOException {
- GetContainerStatusRequest request =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- request.setContainerId(containerID);
- ContainerStatus containerStatus =
- containerManager.getContainerStatus(request).getStatus();
- int timeoutSecs = 0;
+ List<ContainerId> list = new ArrayList<ContainerId>();
+ list.add(containerID);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(list);
+ ContainerStatus containerStatus =
+ containerManager.getContainerStatuses(request).getContainerStatuses()
+ .get(0);
+ int timeoutSecs = 0;
while (!containerStatus.getState().equals(finalState)
&& timeoutSecs++ < timeOutMax) {
Thread.sleep(1000);
LOG.info("Waiting for container to get into state " + finalState
+ ". Current state is " + containerStatus.getState());
- containerStatus = containerManager.getContainerStatus(request).getStatus();
+ containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
}
LOG.info("Container state is " + containerStatus.getState());
Assert.assertEquals("ContainerState is not correct (timedout)",
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Wed Jul 24 03:42:52 2013
@@ -24,6 +24,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -37,9 +38,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -51,6 +56,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.InvalidContainerException;
@@ -63,6 +69,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -88,11 +95,11 @@ public class TestContainerManager extend
super.setup();
}
- private ContainerId createContainerId() {
+ private ContainerId createContainerId(int id) {
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
- ContainerId containerId = ContainerId.newInstance(appAttemptId, 0);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, id);
return containerId;
}
@@ -119,6 +126,14 @@ public class TestContainerManager extend
.getKeyId()));
return ugi;
}
+
+ @Override
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest, NMTokenIdentifier identifier) throws YarnException {
+ if(container == null || container.getUser().equals("Fail")){
+ throw new YarnException("Reject this container");
+ }
+ }
};
}
@@ -138,12 +153,17 @@ public class TestContainerManager extend
// Just do a query for a non-existing container.
boolean throwsException = false;
try {
- GetContainerStatusRequest request =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- ContainerId cId = createContainerId();
- request.setContainerId(cId);
- containerManager.getContainerStatus(request);
- } catch (YarnException e) {
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ ContainerId id =createContainerId(0);
+ containerIds.add(id);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(containerIds);
+ GetContainerStatusesResponse response =
+ containerManager.getContainerStatuses(request);
+ if(response.getFailedRequests().containsKey(id)){
+ throw response.getFailedRequests().get(id).deSerialize();
+ }
+ } catch (Throwable e) {
throwsException = true;
}
Assert.assertTrue(throwsException);
@@ -163,7 +183,7 @@ public class TestContainerManager extend
fileWriter.close();
// ////// Construct the Container-id
- ContainerId cId = createContainerId();
+ ContainerId cId = createContainerId(0);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext =
@@ -182,14 +202,17 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(createContainerToken(cId,
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
- containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
@@ -237,7 +260,7 @@ public class TestContainerManager extend
new File(tmpDir, "start_file.txt").getAbsoluteFile();
// ////// Construct the Container-id
- ContainerId cId = createContainerId();
+ ContainerId cId = createContainerId(0);
if (Shell.WINDOWS) {
fileWriter.println("@echo Hello World!> " + processStartFile);
@@ -272,13 +295,17 @@ public class TestContainerManager extend
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(createContainerToken(cId,
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
- containerManager.startContainer(startRequest);
-
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
Thread.sleep(1000);
@@ -305,18 +332,18 @@ public class TestContainerManager extend
Assert.assertTrue("Process is not alive!",
DefaultContainerExecutor.containerIsAlive(pid));
- StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(cId);
- containerManager.stopContainer(stopRequest);
-
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ containerManager.stopContainers(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
- GetContainerStatusRequest gcsRequest =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
- containerManager.getContainerStatus(gcsRequest).getStatus();
+ containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
ExitCode.TERMINATED.getExitCode();
Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
@@ -325,7 +352,7 @@ public class TestContainerManager extend
Assert.assertFalse("Process is still alive!",
DefaultContainerExecutor.containerIsAlive(pid));
}
-
+
private void testContainerLaunchAndExit(int exitCode) throws IOException,
InterruptedException, YarnException {
@@ -335,7 +362,7 @@ public class TestContainerManager extend
new File(tmpDir, "start_file.txt").getAbsoluteFile();
// ////// Construct the Container-id
- ContainerId cId = createContainerId();
+ ContainerId cId = createContainerId(0);
if (Shell.WINDOWS) {
fileWriter.println("@echo Hello World!> " + processStartFile);
@@ -376,21 +403,26 @@ public class TestContainerManager extend
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(createContainerToken(cId,
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
- containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
- GetContainerStatusRequest gcsRequest =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(cId);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
- containerManager.getContainerStatus(gcsRequest).getStatus();
+ containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
// Verify exit status matches exit state of script
Assert.assertEquals(exitCode,
@@ -439,7 +471,7 @@ public class TestContainerManager extend
fileWriter.close();
// ////// Construct the Container-id
- ContainerId cId = createContainerId();
+ ContainerId cId = createContainerId(0);
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
// ////// Construct the container-spec.
@@ -460,11 +492,17 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
- request.setContainerLaunchContext(containerLaunchContext);
- request.setContainerToken(createContainerToken(cId, DUMMY_RM_IDENTIFIER,
- context.getNodeId(), user, context.getContainerTokenSecretManager()));
- containerManager.startContainer(request);
+
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
@@ -528,29 +566,37 @@ public class TestContainerManager extend
@Test
public void testContainerLaunchFromPreviousRM() throws IOException,
- InterruptedException {
+ InterruptedException, YarnException {
containerManager.start();
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ContainerId cId1 = createContainerId();
- ContainerId cId2 = createContainerId();
+ ContainerId cId1 = createContainerId(0);
+ ContainerId cId2 = createContainerId(0);
containerLaunchContext
.setLocalResources(new HashMap<String, LocalResource>());
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest1.setContainerLaunchContext(containerLaunchContext);
+ StartContainerRequest.newInstance(containerLaunchContext,
+ createContainerToken(cId1,
+ ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(startRequest1);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
- startRequest1.setContainerToken(createContainerToken(cId1,
- ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(),
- user, context.getContainerTokenSecretManager()));
boolean catchException = false;
try {
- containerManager.startContainer(startRequest1);
- } catch (YarnException e) {
+ StartContainersResponse response = containerManager.startContainers(allRequests);
+ if(response.getFailedRequests().containsKey(cId1)) {
+ throw response.getFailedRequests().get(cId1).deSerialize();
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
catchException = true;
Assert.assertTrue(e.getMessage().contains(
"Container " + cId1 + " rejected as it is allocated by a previous RM"));
@@ -563,21 +609,143 @@ public class TestContainerManager extend
// Construct the Container with a RMIdentifier within current RM
StartContainerRequest startRequest2 =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest2.setContainerLaunchContext(containerLaunchContext);
- startRequest2.setContainerToken(createContainerToken(cId2,
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
+ StartContainerRequest.newInstance(containerLaunchContext,
+ createContainerToken(cId2,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list2 = new ArrayList<StartContainerRequest>();
+ list.add(startRequest2);
+ StartContainersRequest allRequests2 =
+ StartContainersRequest.newInstance(list2);
+ containerManager.startContainers(allRequests2);
+
boolean noException = true;
try {
- containerManager.startContainer(startRequest2);
+ containerManager.startContainers(allRequests2);
} catch (YarnException e) {
noException = false;
}
// Verify that startContainer get no YarnException
Assert.assertTrue(noException);
}
-
+
+ @Test
+ public void testMultipleContainersLaunch() throws Exception {
+ containerManager.start();
+
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ for (int i = 0; i < 10; i++) {
+ ContainerId cId = createContainerId(i);
+ long identifier = 0;
+ if ((i & 1) == 0)
+ // container with even id fail
+ identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
+ else
+ identifier = DUMMY_RM_IDENTIFIER;
+ Token containerToken =
+ createContainerToken(cId, identifier, context.getNodeId(), user,
+ context.getContainerTokenSecretManager());
+ StartContainerRequest request =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ containerToken);
+ list.add(request);
+ }
+ StartContainersRequest requestList =
+ StartContainersRequest.newInstance(list);
+
+ StartContainersResponse response =
+ containerManager.startContainers(requestList);
+
+ Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
+ for (ContainerId id : response.getSuccessfullyStartedContainers()) {
+ // Containers with odd id should succeed.
+ Assert.assertEquals(1, id.getId() & 1);
+ }
+ Assert.assertEquals(5, response.getFailedRequests().size());
+ for (Map.Entry<ContainerId, SerializedException> entry : response
+ .getFailedRequests().entrySet()) {
+ // Containers with even id should fail.
+ Assert.assertEquals(0, entry.getKey().getId() & 1);
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains(
+ "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
+ }
+ }
+
+ @Test
+ public void testMultipleContainersStopAndGetStatus() throws Exception {
+ containerManager.start();
+ List<StartContainerRequest> startRequest =
+ new ArrayList<StartContainerRequest>();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ for (int i = 0; i < 10; i++) {
+ ContainerId cId = createContainerId(i);
+ String user = null;
+ if ((i & 1) == 0) {
+ // container with even id fail
+ user = "Fail";
+ } else {
+ user = "Pass";
+ }
+ Token containerToken =
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager());
+ StartContainerRequest request =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ containerToken);
+ startRequest.add(request);
+ containerIds.add(cId);
+ }
+ // start containers
+ StartContainersRequest requestList =
+ StartContainersRequest.newInstance(startRequest);
+ containerManager.startContainers(requestList);
+
+ // Get container statuses
+ GetContainerStatusesRequest statusRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
+ GetContainerStatusesResponse statusResponse =
+ containerManager.getContainerStatuses(statusRequest);
+ Assert.assertEquals(5, statusResponse.getContainerStatuses().size());
+ for (ContainerStatus status : statusResponse.getContainerStatuses()) {
+ // Containers with odd id should succeed
+ Assert.assertEquals(1, status.getContainerId().getId() & 1);
+ }
+ Assert.assertEquals(5, statusResponse.getFailedRequests().size());
+ for (Map.Entry<ContainerId, SerializedException> entry : statusResponse
+ .getFailedRequests().entrySet()) {
+ // Containers with even id should fail.
+ Assert.assertEquals(0, entry.getKey().getId() & 1);
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains("Reject this container"));
+ }
+
+ // stop containers
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ StopContainersResponse stopResponse =
+ containerManager.stopContainers(stopRequest);
+ Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers()
+ .size());
+ for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) {
+ // Containers with odd id should succeed.
+ Assert.assertEquals(1, id.getId() & 1);
+ }
+ Assert.assertEquals(5, stopResponse.getFailedRequests().size());
+ for (Map.Entry<ContainerId, SerializedException> entry : stopResponse
+ .getFailedRequests().entrySet()) {
+ // Containers with even id should fail.
+ Assert.assertEquals(0, entry.getKey().getId() & 1);
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains("Reject this container"));
+ }
+ }
+
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
NodeId nodeId, String user,
NMContainerTokenSecretManager containerTokenSecretManager)
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Wed Jul 24 03:42:52 2013
@@ -42,9 +42,10 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -399,10 +400,14 @@ public class TestContainerLaunch extends
// set up the rest of the container
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(createContainerToken(cId));
- containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ createContainerToken(cId));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
@@ -465,18 +470,20 @@ public class TestContainerLaunch extends
Assert.assertTrue("Process is not alive!",
DefaultContainerExecutor.containerIsAlive(pid));
- StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(cId);
- containerManager.stopContainer(stopRequest);
+ // Now test the stop functionality.
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ containerManager.stopContainers(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
- GetContainerStatusRequest gcsRequest =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
- containerManager.getContainerStatus(gcsRequest).getStatus();
+ containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
ExitCode.TERMINATED.getExitCode();
Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
@@ -544,11 +551,15 @@ public class TestContainerLaunch extends
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Token containerToken = createContainerToken(cId);
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
- containerManager.startContainer(startRequest);
+
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ containerToken);
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
@@ -559,19 +570,22 @@ public class TestContainerLaunch extends
processStartFile.exists());
// Now test the stop functionality.
- StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(cId);
- containerManager.stopContainer(stopRequest);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ containerManager.stopContainers(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
// container stop sends a sigterm followed by a sigkill
- GetContainerStatusRequest gcsRequest =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
+
ContainerStatus containerStatus =
- containerManager.getContainerStatus(gcsRequest).getStatus();
+ containerManager.getContainerStatuses(gcsRequest)
+ .getContainerStatuses().get(0);
Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(),
containerStatus.getExitStatus());
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java Wed Jul 24 03:42:52 2013
@@ -18,8 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.server.api.records.SerializedException;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Wed Jul 24 03:42:52 2013
@@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -1013,8 +1014,8 @@ public class TestResourceLocalizationSer
String localizerId, LocalResourceRequest req) {
LocalizerStatus status = createLocalizerStatus(localizerId);
LocalResourceStatus resourceStatus = new LocalResourceStatusPBImpl();
- resourceStatus.setException(YarnServerBuilderUtils
- .newSerializedException(new YarnException("test")));
+ resourceStatus.setException(SerializedException
+ .newInstance(new YarnException("test")));
resourceStatus.setStatus(ResourceStatusType.FETCH_FAILURE);
resourceStatus.setResource(req);
status.addResourceStatus(resourceStatus);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Wed Jul 24 03:42:52 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -809,14 +810,18 @@ public class TestLogAggregationService e
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(TestContainerManager.createContainerToken(
- cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
- context.getContainerTokenSecretManager()));
- this.containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ TestContainerManager.createContainerToken(
+ cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ this.containerManager.startContainers(allRequests);
+
BaseContainerManagerTest.waitForContainerState(this.containerManager,
cId, ContainerState.COMPLETE);