You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/08/02 08:56:41 UTC
svn commit: r1509562 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-...
Author: vinodkv
Date: Fri Aug 2 06:56:40 2013
New Revision: 1509562
URL: http://svn.apache.org/r1509562
Log:
YARN-903. Changed ContainerManager to suppress unnecessary warnings when stopping already stopped containers. Contributed by Omkar Vinit Joshi.
svn merge --ignore-ancestry -c 1509560 ../../trunk/
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Fri Aug 2 06:56:40 2013
@@ -24,6 +24,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-573. Shared data structures in Public Localizer and Private Localizer
are not Thread safe. (Omkar Vinit Joshi via jlowe)
+ YARN-903. Changed ContainerManager to suppress unnecessary warnings when
+ stopping already stopped containers. (Omkar Vinit Joshi via vinodkv)
+
Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Aug 2 06:56:40 2013
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
@@ -456,4 +457,10 @@ public class NodeManager extends Composi
Configuration conf = new YarnConfiguration();
nodeManager.initAndStartNodeManager(conf, false);
}
+
+ @VisibleForTesting
+ @Private
+ public NodeStatusUpdater getNodeStatusUpdater() {
+ return nodeStatusUpdater;
+ }
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java Fri Aug 2 06:56:40 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
public interface NodeStatusUpdater extends Service {
@@ -28,4 +29,8 @@ public interface NodeStatusUpdater exten
NodeStatus getNodeStatusAndUpdateContainersInContext();
long getRMIdentifier();
+
+ public boolean isContainerRecentlyStopped(ContainerId containerId);
+
+ public void clearFinishedContainersFromCache();
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Aug 2 06:56:40 2013
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -68,6 +69,9 @@ import com.google.common.annotations.Vis
public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdater {
+ public static final String YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS =
+ YarnConfiguration.NM_PREFIX + "duration-to-track-stopped-containers";
+
private static final Log LOG = LogFactory.getLog(NodeStatusUpdaterImpl.class);
private final Object heartbeatMonitor = new Object();
@@ -88,6 +92,10 @@ public class NodeStatusUpdaterImpl exten
private Map<ApplicationId, Long> appTokenKeepAliveMap =
new HashMap<ApplicationId, Long>();
private Random keepAliveDelayRandom = new Random();
+ // It will be used to track recently stopped containers on node manager.
+ private final Map<ContainerId, Long> recentlyStoppedContainers;
+ // Duration for which to track recently stopped container.
+ private long durationToTrackStoppedContainers;
private final NodeHealthCheckerService healthChecker;
private final NodeManagerMetrics metrics;
@@ -103,6 +111,8 @@ public class NodeStatusUpdaterImpl exten
this.context = context;
this.dispatcher = dispatcher;
this.metrics = metrics;
+ this.recentlyStoppedContainers =
+ new LinkedHashMap<ContainerId, Long>();
}
@Override
@@ -129,11 +139,27 @@ public class NodeStatusUpdaterImpl exten
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
+ // Default duration to track stopped containers on nodemanager is 10Min.
+ // This should not be assigned very large value as it will remember all the
+ // containers stopped during that time.
+ durationToTrackStoppedContainers =
+ conf.getLong(YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ 600000);
+ if (durationToTrackStoppedContainers < 0) {
+ String message = "Invalid configuration for "
+ + YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS + " default "
+ + "value is 10Min(600000).";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS + " :"
+ + durationToTrackStoppedContainers);
+ }
+ super.serviceInit(conf);
LOG.info("Initialized nodemanager for " + nodeId + ":" +
" physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
" virtual-cores=" + virtualCores);
-
- super.serviceInit(conf);
}
@Override
@@ -290,7 +316,11 @@ public class NodeStatusUpdaterImpl exten
if (containerStatus.getState() == ContainerState.COMPLETE) {
// Remove
i.remove();
-
+ // Adding to finished containers cache. Cache will keep it around at
+ // least for #durationToTrackStoppedContainers duration. In the
+ // subsequent call to stop container it will get removed from cache.
+ addStoppedContainersToCache(containerId);
+
LOG.info("Removed completed container " + containerId);
}
}
@@ -340,6 +370,46 @@ public class NodeStatusUpdaterImpl exten
}
}
+ public boolean isContainerRecentlyStopped(ContainerId containerId) {
+ synchronized (recentlyStoppedContainers) {
+ return recentlyStoppedContainers.containsKey(containerId);
+ }
+ }
+
+ @Private
+ @VisibleForTesting
+ public void addStoppedContainersToCache(ContainerId containerId) {
+ synchronized (recentlyStoppedContainers) {
+ removeVeryOldStoppedContainersFromCache();
+ recentlyStoppedContainers.put(containerId,
+ System.currentTimeMillis() + durationToTrackStoppedContainers);
+ }
+ }
+
+ @Override
+ public void clearFinishedContainersFromCache() {
+ synchronized (recentlyStoppedContainers) {
+ recentlyStoppedContainers.clear();
+ }
+ }
+
+ @Private
+ @VisibleForTesting
+ public void removeVeryOldStoppedContainersFromCache() {
+ synchronized (recentlyStoppedContainers) {
+ long currentTime = System.currentTimeMillis();
+ Iterator<ContainerId> i =
+ recentlyStoppedContainers.keySet().iterator();
+ while (i.hasNext()) {
+ if (recentlyStoppedContainers.get(i.next()) < currentTime) {
+ i.remove();
+ } else {
+ break;
+ }
+ }
+ }
+ }
+
@Override
public long getRMIdentifier() {
return this.rmIdentifier;
@@ -455,4 +525,6 @@ public class NodeStatusUpdaterImpl exten
new Thread(statusUpdaterRunnable, "Node Status Updater");
statusUpdater.start();
}
+
+
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Fri Aug 2 06:56:40 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -581,17 +582,24 @@ public class ContainerManagerImpl extend
authorizeGetAndStopContainerRequest(containerID, container, true,
nmTokenIdentifier);
- dispatcher.getEventHandler().handle(
- new ContainerKillEvent(containerID,
- "Container killed by the ApplicationMaster."));
+ if (container == null) {
+ if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " is not handled by this NodeManager");
+ }
+ } else {
+ dispatcher.getEventHandler().handle(
+ new ContainerKillEvent(containerID,
+ "Container killed by the ApplicationMaster."));
- NMAuditLogger.logSuccess(container.getUser(),
- AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
- .getApplicationAttemptId().getApplicationId(), containerID);
-
- // TODO: Move this code to appropriate place once kill_container is
- // implemented.
- nodeStatusUpdater.sendOutofBandHeartBeat();
+ NMAuditLogger.logSuccess(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
+ .getApplicationAttemptId().getApplicationId(), containerID);
+
+ // TODO: Move this code to appropriate place once kill_container is
+ // implemented.
+ nodeStatusUpdater.sendOutofBandHeartBeat();
+ }
}
/**
@@ -627,6 +635,15 @@ public class ContainerManagerImpl extend
authorizeGetAndStopContainerRequest(containerID, container, false,
nmTokenIdentifier);
+ if (container == null) {
+ if (nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " was recently stopped on node manager.");
+ } else {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " is not handled by this NodeManager");
+ }
+ }
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
return containerStatus;
@@ -658,17 +675,11 @@ public class ContainerManagerImpl extend
container.getContainerId());
} else {
LOG.warn(identifier.getApplicationAttemptId()
- + " attempted to get get status for non-application container : "
+ + " attempted to get status for non-application container : "
+ container.getContainerId().toString());
}
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " is not started by this application attempt.");
}
- if (container == null) {
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " is not handled by this NodeManager");
- }
}
class ContainerEventDispatcher implements EventHandler<ContainerEvent> {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Aug 2 06:56:40 2013
@@ -426,7 +426,7 @@ public class TestNodeStatusUpdater {
return this.nodeStatusUpdater;
}
- protected MyNodeStatusUpdater3 getNodeStatusUpdater() {
+ public MyNodeStatusUpdater3 getNodeStatusUpdater() {
return this.nodeStatusUpdater;
}
}
@@ -745,6 +745,40 @@ public class TestNodeStatusUpdater {
lfs.delete(new Path(basedir.getPath()), true);
}
+ @Test(timeout = 90000)
+ public void testRecentlyFinishedContainers() throws Exception {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(
+ NodeStatusUpdaterImpl.YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ "10000");
+ nm.init(conf);
+ NodeStatusUpdaterImpl nodeStatusUpdater =
+ (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
+
+
+ nodeStatusUpdater.addStoppedContainersToCache(cId);
+ Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId));
+
+ long time1 = System.currentTimeMillis();
+ int waitInterval = 15;
+ while (waitInterval-- > 0
+ && nodeStatusUpdater.isContainerRecentlyStopped(cId)) {
+ nodeStatusUpdater.removeVeryOldStoppedContainersFromCache();
+ Thread.sleep(1000);
+ }
+ long time2 = System.currentTimeMillis();
+ // By this time the container will be removed from cache. need to verify.
+ Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId));
+ Assert.assertTrue((time2 - time1) >= 10000 && (time2 -time1) <= 250000);
+ }
+
+
+
@Test
public void testNMRegistration() throws InterruptedException {
nm = new NodeManager() {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1509562&r1=1509561&r2=1509562&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Fri Aug 2 06:56:40 2013
@@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -260,6 +262,11 @@ public class TestContainerManagerSecurit
Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, validNMToken, true).contains(sb.toString()));
+ // Container is removed from node manager's memory by this time.
+ // trying to stop the container. It should not throw any exception.
+ testStopContainer(rpc, validAppAttemptId, validNode, validContainerId,
+ validNMToken, false);
+
// Rolling over master key twice so that we can check whether older keys
// are used for authentication.
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
@@ -267,13 +274,25 @@ public class TestContainerManagerSecurit
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
// trying get container status. Now saved nmToken should be used for
- // authentication.
+ // authentication... It should complain saying container was recently
+ // stopped.
+ sb = new StringBuilder("Container ");
+ sb.append(validContainerId);
+ sb.append(" was recently stopped on node manager");
+ Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+ validContainerId, validNMToken, true).contains(sb.toString()));
+
+ // Now lets remove the container from nm-memory
+ nm.getNodeStatusUpdater().clearFinishedContainersFromCache();
+
+ // This should fail as container is removed from recently tracked finished
+ // containers.
sb = new StringBuilder("Container ");
sb.append(validContainerId.toString());
sb.append(" is not handled by this NodeManager");
Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).contains(sb.toString()));
-
+
}
private void waitForContainerToFinishOnNM(ContainerId containerId) {
@@ -315,6 +334,23 @@ public class TestContainerManagerSecurit
Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
== nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
}
+
+ private String testStopContainer(YarnRPC rpc,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
+ try {
+ stopContainer(rpc, nmToken,
+ Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
+ nodeId);
+ if (isExceptionExpected) {
+ fail("Exception was expected!!");
+ }
+ return "";
+ } catch (Exception e) {
+ e.printStackTrace();
+ return e.getMessage();
+ }
+ }
private String testGetContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
@@ -334,7 +370,7 @@ public class TestContainerManagerSecurit
}
}
- protected String testStartContainer(YarnRPC rpc,
+ private String testStartContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
org.apache.hadoop.yarn.api.records.Token containerToken,
org.apache.hadoop.yarn.api.records.Token nmToken,
@@ -352,6 +388,29 @@ public class TestContainerManagerSecurit
}
}
+ private void stopContainer(YarnRPC rpc, Token nmToken,
+ List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
+ NodeId nodeId) throws Exception {
+ StopContainersRequest request =
+ StopContainersRequest.newInstance(containerId);
+ ContainerManagementProtocol proxy = null;
+ try {
+ proxy =
+ getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
+ appAttemptId.toString());
+ StopContainersResponse response = proxy.stopContainers(request);
+ if (response.getFailedRequests() != null &&
+ response.getFailedRequests().containsKey(containerId)) {
+ parseAndThrowException(response.getFailedRequests().get(containerId)
+ .deSerialize());
+ }
+ } catch (Exception e) {
+ if (proxy != null) {
+ rpc.stopProxy(proxy, conf);
+ }
+ }
+ }
+
private void
getContainerStatus(YarnRPC rpc,
org.apache.hadoop.yarn.api.records.Token nmToken,