You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/09/14 00:49:38 UTC
svn commit: r1170378 [7/12] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-clie...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Sep 13 22:49:27 2011
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_BIND_ADDRESS;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_BIND_ADDRESS;
-
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -43,12 +40,9 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorAppFinishedEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorAppStartedEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorContainerFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.event.LogAggregatorEvent;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -65,13 +59,6 @@ public class LogAggregationService exten
Path remoteRootLogDir;
private String nodeFile;
- static final String LOG_COMPRESSION_TYPE = NMConfig.NM_PREFIX
- + "logaggregation.log_compression_type";
- static final String DEFAULT_COMPRESSION_TYPE = "none";
-
- private static final String LOG_RENTENTION_POLICY_CONFIG_KEY =
- NMConfig.NM_PREFIX + "logaggregation.retain-policy";
-
private final ConcurrentMap<ApplicationId, AppLogAggregator> appLogAggregators;
private final ExecutorService threadPool;
@@ -86,17 +73,17 @@ public class LogAggregationService exten
public synchronized void init(Configuration conf) {
this.localRootLogDirs =
- conf.getStrings(NMConfig.NM_LOG_DIR, NMConfig.DEFAULT_NM_LOG_DIR);
+ conf.getStrings(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS);
this.remoteRootLogDir =
- new Path(conf.get(NMConfig.REMOTE_USER_LOG_DIR,
- NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+ new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
super.init(conf);
}
@Override
public synchronized void start() {
String address =
- getConfig().get(NM_BIND_ADDRESS, DEFAULT_NM_BIND_ADDRESS);
+ getConfig().get(YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS);
InetSocketAddress cmBindAddress = NetUtils.createSocketAddr(address);
try {
this.nodeFile =
@@ -185,11 +172,13 @@ public class LogAggregationService exten
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
- if (!this.appLogAggregators.containsKey(containerId.getAppId())) {
+ if (!this.appLogAggregators.containsKey(
+ containerId.getApplicationAttemptId().getApplicationId())) {
throw new YarnException("Application is not initialized yet for "
+ containerId);
}
- this.appLogAggregators.get(containerId.getAppId())
+ this.appLogAggregators.get(
+ containerId.getApplicationAttemptId().getApplicationId())
.startContainerLogAggregation(containerId, exitCode.equals("0"));
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogDumper.java Tue Sep 13 22:49:27 2011
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AggregatedLogFormat.LogReader;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -106,8 +105,8 @@ public class LogDumper extends Configure
return -1;
} else {
Path remoteRootLogDir =
- new Path(getConf().get(NMConfig.REMOTE_USER_LOG_DIR,
- NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+ new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
AggregatedLogFormat.LogReader reader =
new AggregatedLogFormat.LogReader(getConf(),
LogAggregationService.getRemoteNodeLogFileForApp(
@@ -151,8 +150,8 @@ public class LogDumper extends Configure
dumpAllContainersLogs(ApplicationId appId, DataOutputStream out)
throws IOException {
Path remoteRootLogDir =
- new Path(getConf().get(NMConfig.REMOTE_USER_LOG_DIR,
- NMConfig.DEFAULT_REMOTE_APP_LOG_DIR));
+ new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
Path remoteAppLogDir =
LogAggregationService.getRemoteAppLogDir(remoteRootLogDir, appId);
RemoteIterator<FileStatus> nodeFiles =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/event/LogAggregatorContainerFinishedEvent.java Tue Sep 13 22:49:27 2011
@@ -23,10 +23,10 @@ import org.apache.hadoop.yarn.api.record
public class LogAggregatorContainerFinishedEvent extends LogAggregatorEvent {
private final ContainerId containerId;
- private final String exitCode;
+ private final int exitCode;
public LogAggregatorContainerFinishedEvent(ContainerId containerId,
- String exitCode) {
+ int exitCode) {
super(LogAggregatorEventType.CONTAINER_FINISHED);
this.containerId = containerId;
this.exitCode = exitCode;
@@ -36,7 +36,7 @@ public class LogAggregatorContainerFinis
return this.containerId;
}
- public String getExitCode() {
+ public int getExitCode() {
return this.exitCode;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Sep 13 22:49:27 2011
@@ -29,11 +29,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
@@ -45,14 +45,6 @@ public class ContainersMonitorImpl exten
final static Log LOG = LogFactory
.getLog(ContainersMonitorImpl.class);
- private final static String MONITORING_INTERVAL_CONFIG_KEY =
- NMConfig.NM_PREFIX + "containers-monitor.monitoring-interval";
- public static final String RESOURCE_CALCULATOR_PLUGIN_CONFIG_KEY =
- NMConfig.NM_PREFIX + "containers-monitor.resourcecalculatorplugin";
- public static final String NM_RESERVED_PHYSICALMEMORY_MB =
- NMConfig.NM_PREFIX + "reserved-physical-memory.mb";
-
- private final static int MONITORING_INTERVAL_DEFAULT = 3000;
private long monitoringInterval;
private MonitoringThread monitoringThread;
@@ -96,11 +88,11 @@ public class ContainersMonitorImpl exten
@Override
public synchronized void init(Configuration conf) {
this.monitoringInterval =
- conf.getLong(MONITORING_INTERVAL_CONFIG_KEY,
- MONITORING_INTERVAL_DEFAULT);
+ conf.getLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_NM_CONTAINER_MON_INTERVAL_MS);
Class<? extends ResourceCalculatorPlugin> clazz =
- conf.getClass(RESOURCE_CALCULATOR_PLUGIN_CONFIG_KEY, null,
+ conf.getClass(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, null,
ResourceCalculatorPlugin.class);
this.resourceCalculatorPlugin =
ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
@@ -120,7 +112,7 @@ public class ContainersMonitorImpl exten
// ///////// Virtual memory configuration //////
this.maxVmemAllottedForContainers =
- conf.getLong(NMConfig.NM_VMEM_GB, NMConfig.DEFAULT_NM_VMEM_GB);
+ conf.getLong(YarnConfiguration.NM_VMEM_GB, YarnConfiguration.DEFAULT_NM_VMEM_GB);
this.maxVmemAllottedForContainers =
this.maxVmemAllottedForContainers * 1024 * 1024 * 1024L; //Normalize
@@ -131,7 +123,7 @@ public class ContainersMonitorImpl exten
// ///////// Physical memory configuration //////
long reservedPmemOnNM =
- conf.getLong(NM_RESERVED_PHYSICALMEMORY_MB, DISABLED_MEMORY_LIMIT);
+ conf.getLong(YarnConfiguration.NM_RESERVED_MEMORY_MB, DISABLED_MEMORY_LIMIT);
reservedPmemOnNM =
reservedPmemOnNM == DISABLED_MEMORY_LIMIT
? DISABLED_MEMORY_LIMIT
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Tue Sep 13 22:49:27 2011
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOG_DIR;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOG_DIR;
-
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
@@ -33,10 +30,10 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -64,7 +61,7 @@ public class ContainerLogsPage extends N
@Inject
public ContainersLogsBlock(Configuration conf, Context context) {
this.conf = conf;
- this.logsSelector = new LocalDirAllocator(NMConfig.NM_LOG_DIR);
+ this.logsSelector = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
this.nmContext = context;
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
}
@@ -98,7 +95,8 @@ public class ContainerLogsPage extends N
logFile =
new File(this.logsSelector
.getLocalPathToRead(
- ConverterUtils.toString(containerId.getAppId())
+ ConverterUtils.toString(
+ containerId.getApplicationAttemptId().getApplicationId())
+ Path.SEPARATOR + $(CONTAINER_ID)
+ Path.SEPARATOR
+ $(CONTAINER_LOG_TYPE), this.conf).toUri()
@@ -176,10 +174,12 @@ public class ContainerLogsPage extends N
static List<File>
getContainerLogDirs(Configuration conf, ContainerId containerId) {
String[] logDirs =
- conf.getStrings(NM_LOG_DIR, DEFAULT_NM_LOG_DIR);
+ conf.getStrings(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS);
List<File> containerLogDirs = new ArrayList<File>(logDirs.length);
for (String logDir : logDirs) {
- String appIdStr = ConverterUtils.toString(containerId.getAppId());
+ String appIdStr =
+ ConverterUtils.toString(
+ containerId.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
String containerIdStr = ConverterUtils.toString(containerId);
containerLogDirs.add(new File(appLogDir, containerIdStr));
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Tue Sep 13 22:49:27 2011
@@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.uti
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -69,10 +70,14 @@ public class ContainerPage extends NMVie
ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
Container container = this.nmContext.getContainers().get(containerID);
ContainerStatus containerData = container.cloneAndGetContainerStatus();
+ int exitCode = containerData.getExitStatus();
+ String exiStatus =
+ (exitCode == YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS) ?
+ "N/A" : String.valueOf(exitCode);
info("Container information")
._("ContainerID", $(CONTAINER_ID))
._("ContainerState", container.getContainerState())
- ._("ExitStatus", containerData.getExitStatus())
+ ._("ExitStatus", exiStatus)
._("Diagnostics", containerData.getDiagnostics())
._("User", container.getUser())
._("TotalMemoryNeeded",
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java Tue Sep 13 22:49:27 2011
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.util.StringHelper.join;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.webapp.Controller;
import com.google.inject.Inject;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Sep 13 22:49:27 2011
@@ -24,8 +24,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.webapp.WebApp;
@@ -52,8 +52,8 @@ public class WebServer extends AbstractS
@Override
public synchronized void start() {
- String bindAddress = getConfig().get(NMConfig.NM_HTTP_BIND_ADDRESS,
- NMConfig.DEFAULT_NM_HTTP_BIND_ADDRESS);
+ String bindAddress = getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
this.webApp =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Sep 13 22:49:27 2011
@@ -20,12 +20,15 @@ package org.apache.hadoop.yarn.server.no
import static org.junit.Assert.fail;
+import java.util.Collection;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -57,8 +60,8 @@ public class DummyContainerManager exten
public DummyContainerManager(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics) {
- super(context, exec, deletionContext, nodeStatusUpdater, metrics);
+ NodeManagerMetrics metrics, ContainerTokenSecretManager containerTokenSecretManager) {
+ super(context, exec, deletionContext, nodeStatusUpdater, metrics, containerTokenSecretManager);
}
@Override
@@ -79,14 +82,17 @@ public class DummyContainerManager exten
ContainerLocalizationRequestEvent rsrcReqs =
(ContainerLocalizationRequestEvent) event;
// simulate localization of all requested resources
- for (LocalResourceRequest req : rsrcReqs.getRequestedResources()) {
- LOG.info("DEBUG: " + req + ":" +
- rsrcReqs.getContainer().getContainerID());
- dispatcher.getEventHandler().handle(
- new ContainerResourceLocalizedEvent(
- rsrcReqs.getContainer().getContainerID(), req,
- new Path("file:///local" + req.getPath().toUri().getPath())));
- }
+ for (Collection<LocalResourceRequest> rc : rsrcReqs
+ .getRequestedResources().values()) {
+ for (LocalResourceRequest req : rc) {
+ LOG.info("DEBUG: " + req + ":"
+ + rsrcReqs.getContainer().getContainerID());
+ dispatcher.getEventHandler().handle(
+ new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
+ .getContainerID(), req, new Path("file:///local"
+ + req.getPath().toUri().getPath())));
+ }
+ }
break;
case CLEANUP_CONTAINER_RESOURCES:
Container container =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java Tue Sep 13 22:49:27 2011
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.junit.After;
@@ -119,13 +120,13 @@ public class TestContainerManagerWithLCE
private boolean shouldRunTest() {
return System
- .getProperty(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY) != null;
+ .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
}
@Override
protected ContainerExecutor createContainerExecutor() {
- super.conf.set(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY, System
- .getProperty(LinuxContainerExecutor.CONTAINER_EXECUTOR_EXEC_KEY));
+ super.conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, System
+ .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH));
LinuxContainerExecutor linuxContainerExecutor =
new LinuxContainerExecutor();
linuxContainerExecutor.setConf(super.conf);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Sep 13 22:49:27 2011
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.no
import java.io.File;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.NodeHealthCheckerService;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -37,9 +35,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -48,8 +46,8 @@ import org.junit.Test;
public class TestEventFlow {
- private static final Log LOG = LogFactory.getLog(TestEventFlow.class);
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ private static final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
private static File localDir = new File("target",
TestEventFlow.class.getName() + "-localDir").getAbsoluteFile();
@@ -74,9 +72,10 @@ public class TestEventFlow {
Context context = new NMContext();
YarnConfiguration conf = new YarnConfiguration();
- conf.set(NMConfig.NM_LOCAL_DIR, localDir.getAbsolutePath());
- conf.set(NMConfig.NM_LOG_DIR, localLogDir.getAbsolutePath());
- conf.set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ remoteLogDir.getAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(conf);
@@ -84,8 +83,9 @@ public class TestEventFlow {
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = null;
NodeManagerMetrics metrics = NodeManagerMetrics.create();
+ ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager();
NodeStatusUpdater nodeStatusUpdater =
- new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
+ new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -98,27 +98,36 @@ public class TestEventFlow {
};
DummyContainerManager containerManager =
- new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics);
+ new DummyContainerManager(context, exec, del, nodeStatusUpdater,
+ metrics, containerTokenSecretManager);
containerManager.init(conf);
containerManager.start();
- ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ContainerLaunchContext launchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
- cID.setAppId(recordFactory.newRecordInstance(ApplicationId.class));
- ApplicationAttemptId atId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- atId.setApplicationId(cID.getAppId());
- cID.setAppAttemptId(atId);
+ ApplicationId applicationId =
+ recordFactory.newRecordInstance(ApplicationId.class);
+ applicationId.setClusterTimestamp(0);
+ applicationId.setId(0);
+ ApplicationAttemptId applicationAttemptId =
+ recordFactory.newRecordInstance(ApplicationAttemptId.class);
+ applicationAttemptId.setApplicationId(applicationId);
+ applicationAttemptId.setAttemptId(0);
+ cID.setApplicationAttemptId(applicationAttemptId);
launchContext.setContainerId(cID);
launchContext.setUser("testing");
launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
- StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
+ StartContainerRequest request =
+ recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
ContainerState.RUNNING);
- StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
+ StopContainerRequest stopRequest =
+ recordFactory.newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(cID);
containerManager.stopContainer(stopRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Tue Sep 13 22:49:27 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.net.NetUtils;
@@ -211,7 +212,8 @@ public class TestNMAuditLogger {
public void testNMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
- Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
+ Server server = RPC.getServer(TestProtocol.class,
+ new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Sep 13 22:49:27 2011
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.no
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@@ -28,10 +32,13 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.NodeHealthCheckerService;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -47,9 +54,12 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.junit.After;
import org.junit.Assert;
@@ -58,24 +68,38 @@ import org.junit.Test;
public class TestNodeStatusUpdater {
+ // temp fix until metrics system can auto-detect itself running in unit test:
+ static {
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
static final Log LOG = LogFactory.getLog(TestNodeStatusUpdater.class);
static final Path basedir =
new Path("target", TestNodeStatusUpdater.class.getName());
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ private static final RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(null);
int heartBeatID = 0;
volatile Error nmStartError = null;
+ private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
+
+ @After
+ public void tearDown() {
+ this.registeredNodes.clear();
+ DefaultMetricsSystem.shutdown();
+ }
private class MyResourceTracker implements ResourceTracker {
- private Context context;
+ private final Context context;
public MyResourceTracker(Context context) {
this.context = context;
}
@Override
- public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException {
+ public RegisterNodeManagerResponse registerNodeManager(
+ RegisterNodeManagerRequest request) throws YarnRemoteException {
NodeId nodeId = request.getNodeId();
Resource resource = request.getResource();
LOG.info("Registering " + nodeId.toString());
@@ -86,33 +110,60 @@ public class TestNodeStatusUpdater {
Assert.fail(e.getMessage());
}
Assert.assertEquals(5 * 1024, resource.getMemory());
- RegistrationResponse regResponse = recordFactory.newRecordInstance(RegistrationResponse.class);
-
- RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
+ registeredNodes.add(nodeId);
+ RegistrationResponse regResponse = recordFactory
+ .newRecordInstance(RegistrationResponse.class);
+
+ RegisterNodeManagerResponse response = recordFactory
+ .newRecordInstance(RegisterNodeManagerResponse.class);
response.setRegistrationResponse(regResponse);
return response;
}
- ApplicationId applicationID = recordFactory.newRecordInstance(ApplicationId.class);
- ApplicationAttemptId appAttemptID = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- ContainerId firstContainerID = recordFactory.newRecordInstance(ContainerId.class);
- ContainerId secondContainerID = recordFactory.newRecordInstance(ContainerId.class);
+ ApplicationId applicationID = recordFactory
+ .newRecordInstance(ApplicationId.class);
+ ApplicationAttemptId appAttemptID = recordFactory
+ .newRecordInstance(ApplicationAttemptId.class);
+ ContainerId firstContainerID = recordFactory
+ .newRecordInstance(ContainerId.class);
+ ContainerId secondContainerID = recordFactory
+ .newRecordInstance(ContainerId.class);
+
+ private Map<ApplicationId, List<ContainerStatus>> getAppToContainerStatusMap(
+ List<ContainerStatus> containers) {
+ Map<ApplicationId, List<ContainerStatus>> map =
+ new HashMap<ApplicationId, List<ContainerStatus>>();
+ for (ContainerStatus cs : containers) {
+ ApplicationId applicationId =
+ cs.getContainerId().getApplicationAttemptId().getApplicationId();
+ List<ContainerStatus> appContainers = map.get(applicationId);
+ if (appContainers == null) {
+ appContainers = new ArrayList<ContainerStatus>();
+ map.put(applicationId, appContainers);
+ }
+ appContainers.add(cs);
+ }
+ return map;
+ }
@Override
- public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnRemoteException {
+ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
+ throws YarnRemoteException {
NodeStatus nodeStatus = request.getNodeStatus();
LOG.info("Got heartbeat number " + heartBeatID);
nodeStatus.setResponseId(heartBeatID++);
+ Map<ApplicationId, List<ContainerStatus>> appToContainers =
+ getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
if (heartBeatID == 1) {
- Assert.assertEquals(0, nodeStatus.getAllContainers().size());
+ Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
// Give a container to the NM.
applicationID.setId(heartBeatID);
appAttemptID.setApplicationId(applicationID);
- firstContainerID.setAppId(applicationID);
- firstContainerID.setAppAttemptId(appAttemptID);
+ firstContainerID.setApplicationAttemptId(appAttemptID);
firstContainerID.setId(heartBeatID);
- ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ContainerLaunchContext launchContext = recordFactory
+ .newRecordInstance(ContainerLaunchContext.class);
launchContext.setContainerId(firstContainerID);
launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
launchContext.getResource().setMemory(2);
@@ -121,11 +172,9 @@ public class TestNodeStatusUpdater {
} else if (heartBeatID == 2) {
// Checks on the RM end
Assert.assertEquals("Number of applications should only be one!", 1,
- nodeStatus.getAllContainers().size());
+ nodeStatus.getContainersStatuses().size());
Assert.assertEquals("Number of container for the app should be one!",
- 1, nodeStatus.getContainers(applicationID).size());
- Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
- .getResource().getMemory());
+ 1, appToContainers.get(applicationID).size());
// Checks on the NM end
ConcurrentMap<ContainerId, Container> activeContainers =
@@ -135,10 +184,10 @@ public class TestNodeStatusUpdater {
// Give another container to the NM.
applicationID.setId(heartBeatID);
appAttemptID.setApplicationId(applicationID);
- secondContainerID.setAppId(applicationID);
- secondContainerID.setAppAttemptId(appAttemptID);
+ secondContainerID.setApplicationAttemptId(appAttemptID);
secondContainerID.setId(heartBeatID);
- ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ContainerLaunchContext launchContext = recordFactory
+ .newRecordInstance(ContainerLaunchContext.class);
launchContext.setContainerId(secondContainerID);
launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
launchContext.getResource().setMemory(3);
@@ -147,23 +196,21 @@ public class TestNodeStatusUpdater {
} else if (heartBeatID == 3) {
// Checks on the RM end
Assert.assertEquals("Number of applications should only be one!", 1,
- nodeStatus.getAllContainers().size());
+ appToContainers.size());
Assert.assertEquals("Number of container for the app should be two!",
- 2, nodeStatus.getContainers(applicationID).size());
- Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
- .getResource().getMemory());
- Assert.assertEquals(3, nodeStatus.getContainers(applicationID).get(1)
- .getResource().getMemory());
+ 2, appToContainers.get(applicationID).size());
// Checks on the NM end
ConcurrentMap<ContainerId, Container> activeContainers =
this.context.getContainers();
Assert.assertEquals(2, activeContainers.size());
}
- HeartbeatResponse response = recordFactory.newRecordInstance(HeartbeatResponse.class);
+ HeartbeatResponse response = recordFactory
+ .newRecordInstance(HeartbeatResponse.class);
response.setResponseId(heartBeatID);
- NodeHeartbeatResponse nhResponse = recordFactory.newRecordInstance(NodeHeartbeatResponse.class);
+ NodeHeartbeatResponse nhResponse = recordFactory
+ .newRecordInstance(NodeHeartbeatResponse.class);
nhResponse.setHeartbeatResponse(response);
return nhResponse;
}
@@ -173,8 +220,10 @@ public class TestNodeStatusUpdater {
private Context context;
public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
- super(context, dispatcher, healthChecker, metrics);
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
+ ContainerTokenSecretManager containerTokenSecretManager) {
+ super(context, dispatcher, healthChecker, metrics,
+ containerTokenSecretManager);
this.context = context;
}
@@ -200,21 +249,23 @@ public class TestNodeStatusUpdater {
final NodeManager nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
+ ContainerTokenSecretManager containerTokenSecretManager) {
return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
- metrics);
+ metrics, containerTokenSecretManager);
}
};
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(NMConfig.NM_VMEM_GB, 5); // 5GB
- conf.set(NMConfig.NM_BIND_ADDRESS, "127.0.0.1:12345");
- conf.set(NMConfig.NM_LOCALIZER_BIND_ADDRESS, "127.0.0.1:12346");
- conf.set(NMConfig.NM_LOG_DIR, new Path(basedir, "logs").toUri().getPath());
- conf.set(NMConfig.REMOTE_USER_LOG_DIR, new Path(basedir, "remotelogs")
- .toUri().getPath());
- conf.set(NMConfig.NM_LOCAL_DIR, new Path(basedir, "nm0").toUri().getPath());
+ YarnConfiguration conf = createNMConfig();
nm.init(conf);
+
+ // verify that the last service is the nodeStatusUpdater (ie registration
+ // with RM)
+ Object[] services = nm.getServices().toArray();
+ Object lastService = services[services.length-1];
+ Assert.assertTrue("last service is NOT the node status updater",
+ lastService instanceof NodeStatusUpdater);
+
new Thread() {
public void run() {
try {
@@ -244,7 +295,75 @@ public class TestNodeStatusUpdater {
while (heartBeatID <= 3) {
Thread.sleep(500);
}
+ Assert.assertEquals("Number of registered NMs is wrong!!", 1,
+ this.registeredNodes.size());
nm.stop();
}
+
+ /**
+ * Verifies that if for some reason NM fails to start ContainerManager RPC
+ * server, RM is oblivious to NM's presence. The behaviour is like this
+ * because otherwise, NM will report to RM even if all its servers are not
+ * started properly, RM will think that the NM is alive and will retire the NM
+ * only after NM_EXPIRY interval. See MAPREDUCE-2749.
+ */
+ @Test
+ public void testNoRegistrationWhenNMServicesFail() {
+
+ final NodeManager nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
+ ContainerTokenSecretManager containerTokenSecretManager) {
+ return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
+ metrics, containerTokenSecretManager);
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(Context context,
+ ContainerExecutor exec, DeletionService del,
+ NodeStatusUpdater nodeStatusUpdater,
+ ContainerTokenSecretManager containerTokenSecretManager) {
+ return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+ metrics, containerTokenSecretManager) {
+ @Override
+ public void start() {
+ // Simulating failure of starting RPC server
+ throw new YarnException("Starting of RPC Server failed");
+ }
+ };
+ }
+ };
+
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ try {
+ nm.start();
+ Assert.fail("NM should have failed to start. Didn't get exception!!");
+ } catch (Exception e) {
+ Assert.assertEquals("Starting of RPC Server failed", e.getCause()
+ .getMessage());
+ }
+
+ Assert.assertEquals("NM state is wrong!", Service.STATE.INITED, nm
+ .getServiceState());
+
+ Assert.assertEquals("Number of registered nodes is wrong!", 0,
+ this.registeredNodes.size());
+ }
+
+ private YarnConfiguration createNMConfig() {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_VMEM_GB, 5); // 5GB
+ conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+ conf.set(YarnConfiguration.NM_LOG_DIRS, new Path(basedir, "logs").toUri()
+ .getPath());
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, new Path(basedir,
+ "remotelogs").toUri().getPath());
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, new Path(basedir, "nm0")
+ .toUri().getPath());
+ return conf;
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java Tue Sep 13 22:49:27 2011
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import org.apache.avro.ipc.Server;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -39,7 +40,7 @@ public class TestPBLocalizerRPC {
static RecordFactory createPBRecordFactory() {
Configuration conf = new Configuration();
- conf.set(RecordFactoryProvider.RPC_SERIALIZER_KEY, "protocolbuffers");
+ conf.set(YarnConfiguration.IPC_SERIALIZER_TYPE, "protocolbuffers");
return RecordFactoryProvider.getRecordFactory(conf);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java Tue Sep 13 22:49:27 2011
@@ -24,6 +24,7 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
@@ -46,7 +47,7 @@ public class TestPBRecordImpl {
static RecordFactory createPBRecordFactory() {
Configuration conf = new Configuration();
- conf.set(RecordFactoryProvider.RPC_SERIALIZER_KEY, "protocolbuffers");
+ conf.set(YarnConfiguration.IPC_SERIALIZER_TYPE, "protocolbuffers");
return RecordFactoryProvider.getRecordFactory(conf);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Sep 13 22:49:27 2011
@@ -41,13 +41,13 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalRMInterface;
-import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
@@ -68,6 +68,7 @@ public abstract class BaseContainerManag
protected static File localLogDir;
protected static File remoteLogDir;
protected static File tmpDir;
+ protected ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager();
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
@@ -95,7 +96,7 @@ public abstract class BaseContainerManag
protected String user = "nobody";
protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(
- context, new AsyncDispatcher(), null, metrics) {
+ context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -129,10 +130,10 @@ public abstract class BaseContainerManag
LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
String bindAddress = "0.0.0.0:5555";
- conf.set(NMConfig.NM_BIND_ADDRESS, bindAddress);
- conf.set(NMConfig.NM_LOCAL_DIR, localDir.getAbsolutePath());
- conf.set(NMConfig.NM_LOG_DIR, localLogDir.getAbsolutePath());
- conf.set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
// Default delSrvc
delSrvc = new DeletionService(exec) {
@@ -148,7 +149,7 @@ public abstract class BaseContainerManag
exec = createContainerExecutor();
containerManager =
new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
- metrics);
+ metrics, this.containerTokenSecretManager);
containerManager.init(conf);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Tue Sep 13 22:49:27 2011
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -95,10 +96,10 @@ public class TestAuxServices {
@Test
public void testAuxEventDispatch() {
Configuration conf = new Configuration();
- conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
ServiceA.class, Service.class);
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
ServiceB.class, Service.class);
conf.setInt("A.expected.init", 1);
conf.setInt("B.expected.stop", 1);
@@ -123,10 +124,10 @@ public class TestAuxServices {
@Test
public void testAuxServices() {
Configuration conf = new Configuration();
- conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
ServiceA.class, Service.class);
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
ServiceB.class, Service.class);
final AuxServices aux = new AuxServices();
aux.init(conf);
@@ -154,10 +155,10 @@ public class TestAuxServices {
@Test
public void testAuxServicesMeta() {
Configuration conf = new Configuration();
- conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
ServiceA.class, Service.class);
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
ServiceB.class, Service.class);
final AuxServices aux = new AuxServices();
aux.init(conf);
@@ -191,10 +192,10 @@ public class TestAuxServices {
@Test
public void testAuxUnexpectedStop() {
Configuration conf = new Configuration();
- conf.setStrings(AuxServices.AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Asrv"),
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
ServiceA.class, Service.class);
- conf.setClass(String.format(AuxServices.AUX_SERVICE_CLASS_FMT, "Bsrv"),
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
ServiceB.class, Service.class);
final AuxServices aux = new AuxServices();
aux.init(conf);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Sep 13 22:49:27 2011
@@ -23,7 +23,11 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import junit.framework.Assert;
@@ -53,6 +57,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
@@ -66,6 +71,20 @@ public class TestContainerManager extend
LOG = LogFactory.getLog(TestContainerManager.class);
}
+ private ContainerId createContainerId() {
+ ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
+ appId.setClusterTimestamp(0);
+ appId.setId(0);
+ ApplicationAttemptId appAttemptId =
+ recordFactory.newRecordInstance(ApplicationAttemptId.class);
+ appAttemptId.setApplicationId(appId);
+ appAttemptId.setAttemptId(1);
+ ContainerId containerId =
+ recordFactory.newRecordInstance(ContainerId.class);
+ containerId.setApplicationAttemptId(appAttemptId);
+ return containerId;
+ }
+
@Test
public void testContainerManagerInitialization() throws IOException {
@@ -74,14 +93,9 @@ public class TestContainerManager extend
// Just do a query for a non-existing container.
boolean throwsException = false;
try {
- GetContainerStatusRequest request = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
- ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- appAttemptId.setApplicationId(appId);
- appAttemptId.setAttemptId(1);
- ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
- cId.setAppId(appId);
- cId.setAppAttemptId(appAttemptId);
+ GetContainerStatusRequest request =
+ recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+ ContainerId cId = createContainerId();
request.setContainerId(cId);
containerManager.getContainerStatus(request);
} catch (YarnRemoteException e) {
@@ -106,20 +120,14 @@ public class TestContainerManager extend
ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// ////// Construct the Container-id
- ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
- ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- appAttemptId.setApplicationId(appId);
- appAttemptId.setAttemptId(1);
- ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
- cId.setAppId(appId);
- cId.setAppAttemptId(appAttemptId);
+ ContainerId cId = createContainerId();
container.setContainerId(cId);
container.setUser(user);
// ////// Construct the container-spec.
- ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
-// containerLaunchContext.resources = new HashMap<CharSequence, LocalResource>();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(file.getAbsolutePath())));
@@ -130,14 +138,17 @@ public class TestContainerManager extend
rsrc_alpha.setType(LocalResourceType.FILE);
rsrc_alpha.setTimestamp(file.lastModified());
String destinationFile = "dest_file";
- containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
containerLaunchContext.setContainerId(container.getContainerId());
containerLaunchContext.setResource(recordFactory
.newRecordInstance(Resource.class));
-// containerLaunchContext.command = new ArrayList<CharSequence>();
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
+ StartContainerRequest startRequest =
+ recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
containerManager.startContainer(startRequest);
@@ -146,7 +157,7 @@ public class TestContainerManager extend
ContainerState.COMPLETE);
// Now ascertain that the resources are localised correctly.
- // TODO: Don't we need clusterStamp in localDir?
+ ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
String appIDStr = ConverterUtils.toString(appId);
String containerIDStr = ConverterUtils.toString(cId);
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
@@ -186,41 +197,41 @@ public class TestContainerManager extend
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
new File(tmpDir, "start_file.txt").getAbsoluteFile();
- fileWriter.write("\numask 0"); // So that start file is readable by the test.
+ fileWriter.write("\numask 0"); // So that start file is readable by the test
fileWriter.write("\necho Hello World! > " + processStartFile);
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nexec sleep 100");
fileWriter.close();
- ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
// ////// Construct the Container-id
- ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
- ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- appAttemptId.setApplicationId(appId);
- appAttemptId.setAttemptId(1);
- ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
- cId.setAppId(appId);
- cId.setAppAttemptId(appAttemptId);
+ ContainerId cId = createContainerId();
containerLaunchContext.setContainerId(cId);
containerLaunchContext.setUser(user);
-// containerLaunchContext.resources =new HashMap<CharSequence, LocalResource>();
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
- LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
+ LocalResource rsrc_alpha =
+ recordFactory.newRecordInstance(LocalResource.class);
rsrc_alpha.setResource(resource_alpha);
rsrc_alpha.setSize(-1);
rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
rsrc_alpha.setType(LocalResourceType.FILE);
rsrc_alpha.setTimestamp(scriptFile.lastModified());
String destinationFile = "dest_file";
- containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
- containerLaunchContext.addCommand("/bin/bash");
- containerLaunchContext.addCommand(scriptFile.getAbsolutePath());
+ List<String> commands = new ArrayList<String>();
+ commands.add("/bin/bash");
+ commands.add(scriptFile.getAbsolutePath());
+ containerLaunchContext.setCommands(commands);
containerLaunchContext.setResource(recordFactory
.newRecordInstance(Resource.class));
containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
@@ -263,10 +274,12 @@ public class TestContainerManager extend
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
- GetContainerStatusRequest gcsRequest = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
+ GetContainerStatusRequest gcsRequest =
+ recordFactory.newRecordInstance(GetContainerStatusRequest.class);
gcsRequest.setContainerId(cId);
- ContainerStatus containerStatus = containerManager.getContainerStatus(gcsRequest).getStatus();
- Assert.assertEquals(String.valueOf(ExitCode.KILLED.getExitCode()),
+ ContainerStatus containerStatus =
+ containerManager.getContainerStatus(gcsRequest).getStatus();
+ Assert.assertEquals(ExitCode.KILLED.getExitCode(),
containerStatus.getExitStatus());
// Assert that the process is not alive anymore
@@ -281,8 +294,10 @@ public class TestContainerManager extend
// Real del service
delSrvc = new DeletionService(exec);
delSrvc.init(conf);
+ ContainerTokenSecretManager containerTokenSecretManager = new
+ ContainerTokenSecretManager();
containerManager = new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics);
+ nodeStatusUpdater, metrics, containerTokenSecretManager);
containerManager.init(conf);
containerManager.start();
@@ -297,13 +312,8 @@ public class TestContainerManager extend
ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// ////// Construct the Container-id
- ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
- ApplicationAttemptId appAttemptId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
- appAttemptId.setApplicationId(appId);
- appAttemptId.setAttemptId(1);
- ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
- cId.setAppId(appId);
- cId.setAppAttemptId(appAttemptId);
+ ContainerId cId = createContainerId();
+ ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
container.setContainerId(cId);
container.setUser(user);
@@ -322,7 +332,10 @@ public class TestContainerManager extend
rsrc_alpha.setType(LocalResourceType.FILE);
rsrc_alpha.setTimestamp(file.lastModified());
String destinationFile = "dest_file";
- containerLaunchContext.setLocalResource(destinationFile, rsrc_alpha);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
containerLaunchContext.setContainerId(container.getContainerId());
containerLaunchContext.setResource(recordFactory
@@ -337,7 +350,8 @@ public class TestContainerManager extend
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
ContainerState.COMPLETE);
- BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getAppId(),
+ BaseContainerManagerTest.waitForApplicationState(containerManager,
+ cId.getApplicationAttemptId().getApplicationId(),
ApplicationState.RUNNING);
// Now ascertain that the resources are localised correctly.
@@ -369,7 +383,8 @@ public class TestContainerManager extend
containerManager.handle(new CMgrCompletedAppsEvent(Arrays
.asList(new ApplicationId[] { appId })));
- BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getAppId(),
+ BaseContainerManagerTest.waitForApplicationState(containerManager,
+ cId.getApplicationAttemptId().getApplicationId(),
ApplicationState.FINISHED);
// Now ascertain that the resources are localised correctly.