You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/07/24 05:42:54 UTC
svn commit: r1506392 [3/3] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Wed Jul 24 03:42:52 2013
@@ -40,8 +40,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -227,9 +228,6 @@ public class TestContainersMonitor exten
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
- StartContainerRequest startRequest =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
ContainerTokenIdentifier containerIdentifier =
new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER);
@@ -237,8 +235,14 @@ public class TestContainersMonitor exten
BuilderUtils.newContainerToken(context.getNodeId(),
containerManager.getContext().getContainerTokenSecretManager()
.createPassword(containerIdentifier), containerIdentifier);
- startRequest.setContainerToken(containerToken);
- containerManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(containerLaunchContext,
+ containerToken);
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
@@ -260,11 +264,12 @@ public class TestContainersMonitor exten
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE, 60);
- GetContainerStatusRequest gcsRequest =
- recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(cId);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
- containerManager.getContainerStatus(gcsRequest).getStatus();
+ containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
Assert.assertEquals(ExitCode.TERMINATED.getExitCode(),
containerStatus.getExitStatus());
String expectedMsgPattern =
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Wed Jul 24 03:42:52 2013
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -33,11 +35,15 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
@@ -46,8 +52,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -68,8 +72,6 @@ public class AMLauncher implements Runna
private final RMAppAttempt application;
private final Configuration conf;
- private final RecordFactory recordFactory =
- RecordFactoryProvider.getRecordFactory(null);
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
@@ -102,22 +104,42 @@ public class AMLauncher implements Runna
+ " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
createAMContainerLaunchContext(applicationContext, masterContainerID);
- StartContainerRequest request =
- recordFactory.newRecordInstance(StartContainerRequest.class);
- request.setContainerLaunchContext(launchContext);
- request.setContainerToken(masterContainer.getContainerToken());
- containerMgrProxy.startContainer(request);
- LOG.info("Done launching container " + masterContainer
- + " for AM " + application.getAppAttemptId());
+
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(launchContext,
+ masterContainer.getContainerToken());
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+
+ StartContainersResponse response =
+ containerMgrProxy.startContainers(allRequests);
+ if (response.getFailedRequests() != null
+ && response.getFailedRequests().containsKey(masterContainerID)) {
+ Throwable t =
+ response.getFailedRequests().get(masterContainerID).deSerialize();
+ parseAndThrowException(t);
+ } else {
+ LOG.info("Done launching container " + masterContainer + " for AM "
+ + application.getAppAttemptId());
+ }
}
private void cleanup() throws IOException, YarnException {
connect();
ContainerId containerId = masterContainer.getId();
- StopContainerRequest stopRequest =
- recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(containerId);
- containerMgrProxy.stopContainer(stopRequest);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(containerId);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ StopContainersResponse response =
+ containerMgrProxy.stopContainers(stopRequest);
+ if (response.getFailedRequests() != null
+ && response.getFailedRequests().containsKey(containerId)) {
+ Throwable t = response.getFailedRequests().get(containerId).deSerialize();
+ parseAndThrowException(t);
+ }
}
// Protected. For tests.
@@ -254,4 +276,15 @@ public class AMLauncher implements Runna
break;
}
}
+
+ private void parseAndThrowException(Throwable t) throws YarnException,
+ IOException {
+ if (t instanceof YarnException) {
+ throw (YarnException) t;
+ } else if (t instanceof InvalidToken) {
+ throw (InvalidToken) t;
+ } else {
+ throw (IOException) t;
+ }
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Wed Jul 24 03:42:52 2013
@@ -35,7 +35,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -212,9 +213,11 @@ public class Application {
NodeManager nodeManager = task.getNodeManager();
ContainerId containerId = task.getContainerId();
task.stop();
- StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(containerId);
- nodeManager.stopContainer(stopRequest);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(containerId);
+ StopContainersRequest stopRequest =
+ StopContainersRequest.newInstance(containerIds);
+ nodeManager.stopContainers(stopRequest);
Resources.subtractFrom(used, requestSpec.get(task.getPriority()));
@@ -339,10 +342,15 @@ public class Application {
updateResourceRequests(requests.get(priority), type, task);
// Launch the container
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(createCLC());
- startRequest.setContainerToken(container.getContainerToken());
- nodeManager.startContainer(startRequest);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(createCLC(),
+ container.getContainerToken());
+ List<StartContainerRequest> list =
+ new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ nodeManager.startContainers(allRequests);
break;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Wed Jul 24 03:42:52 2013
@@ -31,12 +31,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -158,61 +159,60 @@ public class NodeManager implements Cont
}
@Override
- synchronized public StartContainerResponse startContainer(
- StartContainerRequest request)
+ synchronized public StartContainersResponse startContainers(
+ StartContainersRequest requests)
throws YarnException {
- Token containerToken = request.getContainerToken();
- ContainerTokenIdentifier tokenId = null;
+ for (StartContainerRequest request : requests.getStartContainerRequests()) {
+ Token containerToken = request.getContainerToken();
+ ContainerTokenIdentifier tokenId = null;
+
+ try {
+ tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
- try {
- tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
- }
+ ContainerId containerID = tokenId.getContainerID();
+ ApplicationId applicationId =
+ containerID.getApplicationAttemptId().getApplicationId();
+
+ List<Container> applicationContainers = containers.get(applicationId);
+ if (applicationContainers == null) {
+ applicationContainers = new ArrayList<Container>();
+ containers.put(applicationId, applicationContainers);
+ }
- ContainerId containerID = tokenId.getContainerID();
- ApplicationId applicationId =
- containerID.getApplicationAttemptId().getApplicationId();
-
- List<Container> applicationContainers = containers.get(applicationId);
- if (applicationContainers == null) {
- applicationContainers = new ArrayList<Container>();
- containers.put(applicationId, applicationContainers);
- }
-
- // Sanity check
- for (Container container : applicationContainers) {
- if (container.getId().compareTo(containerID)
- == 0) {
- throw new IllegalStateException(
- "Container " + containerID +
- " already setup on node " + containerManagerAddress);
+ // Sanity check
+ for (Container container : applicationContainers) {
+ if (container.getId().compareTo(containerID) == 0) {
+ throw new IllegalStateException("Container " + containerID
+ + " already setup on node " + containerManagerAddress);
+ }
}
- }
- Container container =
- BuilderUtils.newContainer(containerID,
- this.nodeId, nodeHttpAddress,
- tokenId.getResource(),
- null, null // DKDC - Doesn't matter
+ Container container =
+ BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress,
+ tokenId.getResource(), null, null // DKDC - Doesn't matter
);
- ContainerStatus containerStatus =
- BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW,
- "", -1000);
- applicationContainers.add(container);
- containerStatusMap.put(container, containerStatus);
- Resources.subtractFrom(available, tokenId.getResource());
- Resources.addTo(used, tokenId.getResource());
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("startContainer:" + " node=" + containerManagerAddress
- + " application=" + applicationId + " container=" + container
- + " available=" + available + " used=" + used);
- }
+ ContainerStatus containerStatus =
+ BuilderUtils.newContainerStatus(container.getId(),
+ ContainerState.NEW, "", -1000);
+ applicationContainers.add(container);
+ containerStatusMap.put(container, containerStatus);
+ Resources.subtractFrom(available, tokenId.getResource());
+ Resources.addTo(used, tokenId.getResource());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("startContainer:" + " node=" + containerManagerAddress
+ + " application=" + applicationId + " container=" + container
+ + " available=" + available + " used=" + used);
+ }
- StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class);
+ }
+ StartContainersResponse response =
+ StartContainersResponse.newInstance(null, null, null);
return response;
}
@@ -225,76 +225,79 @@ public class NodeManager implements Cont
}
@Override
- synchronized public StopContainerResponse stopContainer(StopContainerRequest request)
+ synchronized public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
- ContainerId containerID = request.getContainerId();
- String applicationId = String.valueOf(
- containerID.getApplicationAttemptId().getApplicationId().getId());
-
- // Mark the container as COMPLETE
- List<Container> applicationContainers = containers.get(applicationId);
- for (Container c : applicationContainers) {
- if (c.getId().compareTo(containerID) == 0) {
- ContainerStatus containerStatus = containerStatusMap.get(c);
- containerStatus.setState(ContainerState.COMPLETE);
- containerStatusMap.put(c, containerStatus);
+ for (ContainerId containerID : request.getContainerIds()) {
+ String applicationId =
+ String.valueOf(containerID.getApplicationAttemptId()
+ .getApplicationId().getId());
+
+ // Mark the container as COMPLETE
+ List<Container> applicationContainers = containers.get(applicationId);
+ for (Container c : applicationContainers) {
+ if (c.getId().compareTo(containerID) == 0) {
+ ContainerStatus containerStatus = containerStatusMap.get(c);
+ containerStatus.setState(ContainerState.COMPLETE);
+ containerStatusMap.put(c, containerStatus);
+ }
}
- }
-
- // Send a heartbeat
- try {
- heartbeat();
- } catch (IOException ioe) {
- throw RPCUtil.getRemoteException(ioe);
- }
-
- // Remove container and update status
- int ctr = 0;
- Container container = null;
- for (Iterator<Container> i=applicationContainers.iterator(); i.hasNext();) {
- container = i.next();
- if (container.getId().compareTo(containerID) == 0) {
- i.remove();
- ++ctr;
+
+ // Send a heartbeat
+ try {
+ heartbeat();
+ } catch (IOException ioe) {
+ throw RPCUtil.getRemoteException(ioe);
}
- }
-
- if (ctr != 1) {
- throw new IllegalStateException("Container " + containerID +
- " stopped " + ctr + " times!");
- }
-
- Resources.addTo(available, container.getResource());
- Resources.subtractFrom(used, container.getResource());
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("stopContainer:" + " node=" + containerManagerAddress
- + " application=" + applicationId + " container=" + containerID
- + " available=" + available + " used=" + used);
- }
- StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class);
- return response;
+ // Remove container and update status
+ int ctr = 0;
+ Container container = null;
+ for (Iterator<Container> i = applicationContainers.iterator(); i
+ .hasNext();) {
+ container = i.next();
+ if (container.getId().compareTo(containerID) == 0) {
+ i.remove();
+ ++ctr;
+ }
+ }
+
+ if (ctr != 1) {
+ throw new IllegalStateException("Container " + containerID
+ + " stopped " + ctr + " times!");
+ }
+
+ Resources.addTo(available, container.getResource());
+ Resources.subtractFrom(used, container.getResource());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("stopContainer:" + " node=" + containerManagerAddress
+ + " application=" + applicationId + " container=" + containerID
+ + " available=" + available + " used=" + used);
+ }
+ }
+ return StopContainersResponse.newInstance(null,null);
}
@Override
- synchronized public GetContainerStatusResponse getContainerStatus(GetContainerStatusRequest request) throws YarnException {
- ContainerId containerId = request.getContainerId();
- List<Container> appContainers =
- containers.get(
- containerId.getApplicationAttemptId().getApplicationId());
- Container container = null;
- for (Container c : appContainers) {
- if (c.getId().equals(containerId)) {
- container = c;
+ synchronized public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException {
+ List<ContainerStatus> statuses = new ArrayList<ContainerStatus>();
+ for (ContainerId containerId : request.getContainerIds()) {
+ List<Container> appContainers =
+ containers.get(containerId.getApplicationAttemptId()
+ .getApplicationId());
+ Container container = null;
+ for (Container c : appContainers) {
+ if (c.getId().equals(containerId)) {
+ container = c;
+ }
+ }
+ if (container != null
+ && containerStatusMap.get(container).getState() != null) {
+ statuses.add(containerStatusMap.get(container));
}
}
- GetContainerStatusResponse response =
- recordFactory.newRecordInstance(GetContainerStatusResponse.class);
- if (container != null && containerStatusMap.get(container).getState() != null) {
- response.setStatus(containerStatusMap.get(container));
- }
- return response;
+ return GetContainerStatusesResponse.newInstance(statuses, null);
}
public static org.apache.hadoop.yarn.server.api.records.NodeStatus
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Wed Jul 24 03:42:52 2013
@@ -36,14 +36,14 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -98,23 +98,23 @@ public class TestAMAuthorization {
}
@Override
- public StartContainerResponse
- startContainer(StartContainerRequest request)
+ public StartContainersResponse
+ startContainers(StartContainersRequest request)
throws YarnException {
- containerTokens = request.getContainerLaunchContext().getTokens();
- return null;
+ containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
+ return StartContainersResponse.newInstance(null, null, null);
}
@Override
- public StopContainerResponse stopContainer(StopContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
- return null;
+ return StopContainersResponse.newInstance(null, null);
}
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException {
- return null;
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException {
+ return GetContainerStatusesResponse.newInstance(null, null);
}
public Credentials getContainerCredentials() throws IOException {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Wed Jul 24 03:42:52 2013
@@ -19,7 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -27,16 +30,18 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -69,9 +74,10 @@ public class TestApplicationMasterLaunch
int maxAppAttempts;
@Override
- public StartContainerResponse
- startContainer(StartContainerRequest request)
+ public StartContainersResponse
+ startContainers(StartContainersRequest requests)
throws YarnException {
+ StartContainerRequest request = requests.getStartContainerRequests().get(0);
LOG.info("Container started by MyContainerManager: " + request);
launched = true;
Map<String, String> env =
@@ -95,11 +101,13 @@ public class TestApplicationMasterLaunch
Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
maxAppAttempts =
Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
- return null;
+ return StartContainersResponse.newInstance(
+ new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
+ new HashMap<ContainerId, SerializedException>());
}
@Override
- public StopContainerResponse stopContainer(StopContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
LOG.info("Container cleaned up by MyContainerManager");
cleanedup = true;
@@ -107,11 +115,10 @@ public class TestApplicationMasterLaunch
}
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException {
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException {
return null;
}
-
}
@Test
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Wed Jul 24 03:42:52 2013
@@ -19,7 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.annotation.Annotation;
@@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.Contai
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -158,6 +162,9 @@ public class TestClientToAMTokens {
ContainerManagementProtocol containerManager =
mock(ContainerManagementProtocol.class);
+ StartContainersResponse mockResponse = mock(StartContainersResponse.class);
+ when(containerManager.startContainers((StartContainersRequest) any()))
+ .thenReturn(mockResponse);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1506392&r1=1506391&r2=1506392&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed Jul 24 03:42:52 2013
@@ -23,6 +23,8 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.List;
import junit.framework.Assert;
@@ -32,15 +34,20 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -328,18 +335,21 @@ public class TestContainerManagerSecurit
ContainerId containerId,
ApplicationAttemptId appAttemptId, NodeId nodeId,
boolean isExceptionExpected) throws Exception {
- GetContainerStatusRequest request =
- Records.newRecord(GetContainerStatusRequest.class);
- request.setContainerId(containerId);
-
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(containerId);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerManagementProtocol proxy = null;
-
try {
proxy =
getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
appAttemptId.toString());
- proxy.getContainerStatus(request);
-
+ GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
+ if (statuses.getFailedRequests() != null
+ && statuses.getFailedRequests().containsKey(containerId)) {
+ parseAndThrowException(statuses.getFailedRequests().get(containerId)
+ .deSerialize());
+ }
} finally {
if (proxy != null) {
rpc.stopProxy(proxy, conf);
@@ -352,17 +362,21 @@ public class TestContainerManagerSecurit
org.apache.hadoop.yarn.api.records.Token containerToken,
NodeId nodeId, String user) throws Exception {
- StartContainerRequest request =
- Records.newRecord(StartContainerRequest.class);
- request.setContainerToken(containerToken);
ContainerLaunchContext context =
Records.newRecord(ContainerLaunchContext.class);
- request.setContainerLaunchContext(context);
-
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(context,containerToken);
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
ContainerManagementProtocol proxy = null;
try {
proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
- proxy.startContainer(request);
+ StartContainersResponse response = proxy.startContainers(allRequests);
+ for(SerializedException ex : response.getFailedRequests().values()){
+ parseAndThrowException(ex.deSerialize());
+ }
} finally {
if (proxy != null) {
rpc.stopProxy(proxy, conf);
@@ -370,6 +384,17 @@ public class TestContainerManagerSecurit
}
}
+ private void parseAndThrowException(Throwable t) throws YarnException,
+ IOException {
+ if (t instanceof YarnException) {
+ throw (YarnException) t;
+ } else if (t instanceof InvalidToken) {
+ throw (InvalidToken) t;
+ } else {
+ throw (IOException) t;
+ }
+ }
+
protected ContainerManagementProtocol getContainerManagementProtocolProxy(
final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken,
NodeId nodeId, String user) {