You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2012/12/03 23:43:11 UTC
svn commit: r1416712 - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server...
Author: suresh
Date: Mon Dec 3 22:43:00 2012
New Revision: 1416712
URL: http://svn.apache.org/viewvc?rev=1416712&view=rev
Log:
Reverting the previous merge r1416603 which committed some extra changes
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Mon Dec 3 22:43:00 2012
@@ -117,11 +117,7 @@ Release 2.0.3-alpha - Unreleased
YARN-229. Remove old unused RM recovery code. (Bikas Saha via acmurthy)
- YARN-187. Add hierarchical queues to the fair scheduler.
- (Sandy Ryza via tomwhite)
-
- YARN-72. NM should handle cleaning up containers when it shuts down.
- (Sandy Ryza via tomwhite)
+ YARN-187. Add hierarchical queues to the fair scheduler. (Sandy Ryza via tomwhite)
Release 2.0.2-alpha - 2012-09-07
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrCompletedContainersEvent.java Mon Dec 3 22:43:00 2012
@@ -25,23 +25,13 @@ import org.apache.hadoop.yarn.api.record
public class CMgrCompletedContainersEvent extends ContainerManagerEvent {
private List<ContainerId> containerToCleanup;
- private Reason reason;
-
- public CMgrCompletedContainersEvent(List<ContainerId> containersToCleanup, Reason reason) {
+
+ public CMgrCompletedContainersEvent(List<ContainerId> containersToCleanup) {
super(ContainerManagerEventType.FINISH_CONTAINERS);
this.containerToCleanup = containersToCleanup;
- this.reason = reason;
}
public List<ContainerId> getContainersToCleanup() {
return this.containerToCleanup;
}
-
- public Reason getReason() {
- return reason;
- }
-
- public static enum Reason {
- ON_SHUTDOWN, BY_RESOURCEMANAGER
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Mon Dec 3 22:43:00 2012
@@ -19,9 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -64,24 +61,14 @@ public class NodeManager extends Composi
* Priority of the NodeManager shutdown hook.
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
-
- /**
- * Extra duration to wait for containers to be killed on shutdown.
- */
- private static final int SHUTDOWN_CLEANUP_SLOP_MS = 1000;
private static final Log LOG = LogFactory.getLog(NodeManager.class);
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
private ApplicationACLsManager aclsManager;
private NodeHealthCheckerService nodeHealthChecker;
private LocalDirsHandlerService dirsHandler;
- private Context context;
- private AsyncDispatcher dispatcher;
- private ContainerManagerImpl containerManager;
private static CompositeServiceShutdownHook nodeManagerShutdownHook;
- private long waitForContainersOnShutdownMillis;
-
public NodeManager() {
super(NodeManager.class.getName());
}
@@ -128,7 +115,7 @@ public class NodeManager extends Composi
containerTokenSecretManager = new NMContainerTokenSecretManager(conf);
}
- this.context = new NMContext(containerTokenSecretManager);
+ Context context = new NMContext(containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
@@ -144,7 +131,7 @@ public class NodeManager extends Composi
addService(del);
// NodeManager level dispatcher
- this.dispatcher = new AsyncDispatcher();
+ AsyncDispatcher dispatcher = new AsyncDispatcher();
nodeHealthChecker = new NodeHealthCheckerService();
addService(nodeHealthChecker);
@@ -157,7 +144,7 @@ public class NodeManager extends Composi
NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
addService(nodeResourceMonitor);
- containerManager =
+ ContainerManagerImpl containerManager =
createContainerManager(context, exec, del, nodeStatusUpdater,
this.aclsManager, dirsHandler);
addService(containerManager);
@@ -168,20 +155,13 @@ public class NodeManager extends Composi
dispatcher.register(ContainerManagerEventType.class, containerManager);
addService(dispatcher);
-
+
DefaultMetricsSystem.initialize("NodeManager");
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
addService(nodeStatusUpdater);
-
- waitForContainersOnShutdownMillis =
- conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
- YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS) +
- conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
- YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS) +
- SHUTDOWN_CLEANUP_SLOP_MS;
-
+
super.init(conf);
// TODO add local dirs to del
}
@@ -198,44 +178,9 @@ public class NodeManager extends Composi
@Override
public void stop() {
- cleanupContainers();
super.stop();
DefaultMetricsSystem.shutdown();
}
-
- @SuppressWarnings("unchecked")
- private void cleanupContainers() {
- Map<ContainerId, Container> containers = context.getContainers();
- if (containers.isEmpty()) {
- return;
- }
- LOG.info("Containers still running on shutdown: " + containers.keySet());
-
- List<ContainerId> containerIds = new ArrayList<ContainerId>(containers.keySet());
- dispatcher.getEventHandler().handle(
- new CMgrCompletedContainersEvent(containerIds,
- CMgrCompletedContainersEvent.Reason.ON_SHUTDOWN));
-
- LOG.info("Waiting for containers to be killed");
-
- long waitStartTime = System.currentTimeMillis();
- while (!containers.isEmpty() &&
- System.currentTimeMillis() - waitStartTime < waitForContainersOnShutdownMillis) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ex) {
- LOG.warn("Interrupted while sleeping on container kill", ex);
- }
- }
-
- // All containers killed
- if (containers.isEmpty()) {
- LOG.info("All containers in DONE state");
- } else {
- LOG.info("Done waiting for containers to be killed. Still alive: " +
- containers.keySet());
- }
- }
public static class NMContext implements Context {
@@ -337,11 +282,6 @@ public class NodeManager extends Composi
NodeManager createNewNodeManager() {
return new NodeManager();
}
-
- // For testing
- ContainerManagerImpl getContainerManager() {
- return containerManager;
- }
public static void main(String[] args) {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Mon Dec 3 22:43:00 2012
@@ -363,8 +363,7 @@ public class NodeStatusUpdaterImpl exten
.getContainersToCleanupList();
if (containersToCleanup.size() != 0) {
dispatcher.getEventHandler().handle(
- new CMgrCompletedContainersEvent(containersToCleanup,
- CMgrCompletedContainersEvent.Reason.BY_RESOURCEMANAGER));
+ new CMgrCompletedContainersEvent(containersToCleanup));
}
List<ApplicationId> appsToCleanup =
response.getApplicationsToCleanupList();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1416712&r1=1416711&r2=1416712&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Mon Dec 3 22:43:00 2012
@@ -23,8 +23,6 @@ import static org.apache.hadoop.yarn.ser
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -595,16 +593,9 @@ public class ContainerManagerImpl extend
(CMgrCompletedContainersEvent) event;
for (ContainerId container : containersFinishedEvent
.getContainersToCleanup()) {
- String diagnostic = "";
- if (containersFinishedEvent.getReason() ==
- CMgrCompletedContainersEvent.Reason.ON_SHUTDOWN) {
- diagnostic = "Container Killed on Shutdown";
- } else if (containersFinishedEvent.getReason() ==
- CMgrCompletedContainersEvent.Reason.BY_RESOURCEMANAGER) {
- diagnostic = "Container Killed by ResourceManager";
- }
this.dispatcher.getEventHandler().handle(
- new ContainerKillEvent(container, diagnostic));
+ new ContainerKillEvent(container,
+ "Container Killed by ResourceManager"));
}
break;
default: