You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/05/02 19:30:04 UTC
svn commit: r1098673 - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
mr-client/hadoop-mapre...
Author: vinodkv
Date: Mon May 2 17:30:03 2011
New Revision: 1098673
URL: http://svn.apache.org/viewvc?rev=1098673&view=rev
Log:
Moving container logs out of container local-dir into a separate directory structure. Contributed by Vinod Kumar Vavilapalli.
Added:
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerTags.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/container-log4j.properties
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/container_manager.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Mon May 2 17:30:03 2011
@@ -3,6 +3,8 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
MAPREDUCE-279
+ Moving userlogs out of container work-dir into a separate directory structure. (vinodkv)
+
Implement Job Acls in MR Application Master. (sharad)
Implement 'delay scheduling' for better locality in CapacityScheduler and
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Mon May 2 17:30:03 2011
@@ -20,7 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.File;
import java.net.InetSocketAddress;
-import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Vector;
@@ -29,8 +29,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.TaskLog.LogName;
import org.apache.hadoop.mapreduce.ID;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
public class MapReduceChildJVM {
@@ -39,25 +37,8 @@ public class MapReduceChildJVM {
private static final Log LOG = LogFactory.getLog(MapReduceChildJVM.class);
- private static File getUserLogDir(String baseLogDir) {
- return new File(baseLogDir, TaskLog.USERLOGS_DIR_NAME);
- }
-
- private static File getJobDir(String baseLogDir, JobID jobid) {
- return new File(getUserLogDir(baseLogDir), jobid.toString());
- }
-
- private static File getAttemptDir(String baseLogDir, TaskAttemptID taskid,
- boolean isCleanup) {
- String cleanupSuffix = isCleanup ? ".cleanup" : "";
- return new File(getJobDir(baseLogDir, taskid.getJobID()), taskid
- + cleanupSuffix);
- }
-
- private static File getTaskLogFile(String baseLogDir, TaskAttemptID taskid,
- boolean isCleanup, LogName filter) {
- return new File(getAttemptDir(baseLogDir, taskid, isCleanup),
- filter.toString());
+ private static File getTaskLogFile(String logDir, LogName filter) {
+ return new File(logDir, filter.toString());
}
private static String getChildEnv(JobConf jobConf, boolean isMap) {
@@ -69,9 +50,9 @@ public class MapReduceChildJVM {
jobConf.get(jobConf.MAPRED_TASK_ENV));
}
- public static void setVMEnv(Map<String,String> env,
- List<String> classPaths, String pwd, String nmLdLibraryPath, Task task,
- CharSequence applicationTokensFile) {
+ public static void setVMEnv(Map<String, String> env,
+ List<String> classPaths, String pwd, String containerLogDir,
+ String nmLdLibraryPath, Task task, CharSequence applicationTokensFile) {
JobConf conf = task.conf;
@@ -94,7 +75,7 @@ public class MapReduceChildJVM {
/////// Environmental variable LD_LIBRARY_PATH
// for the child of task jvm, set hadoop.root.logger
- env.put("HADOOP_ROOT_LOGGER", "INFO,TLA");
+ env.put("HADOOP_ROOT_LOGGER", "DEBUG,CLA"); // TODO: Debug
// TODO: The following is useful for instance in streaming tasks. Should be
// set in ApplicationMaster's env by the RM.
@@ -107,9 +88,13 @@ public class MapReduceChildJVM {
// FIXME: don't think this is also needed given we already set java
// properties.
long logSize = TaskLog.getTaskLogLength(conf);
- hadoopClientOpts = hadoopClientOpts + "-Dhadoop.tasklog.taskid=" + task.getTaskID()
- + " -Dhadoop.tasklog.iscleanup=" + task.isTaskCleanupTask()
- + " -Dhadoop.tasklog.totalLogFileSize=" + logSize;
+ Vector<String> logProps = new Vector<String>(4);
+ setupLog4jProperties(logProps, logSize, containerLogDir);
+ Iterator<String> it = logProps.iterator();
+ while (it.hasNext()) {
+ hadoopClientOpts += " " + it.next();
+ }
+
env.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
// add the env variables passed by the user
@@ -163,12 +148,10 @@ public class MapReduceChildJVM {
}
private static void setupLog4jProperties(Vector<String> vargs,
- long logSize, String hadoopLogDir, Task task) {
- vargs.add("-Dhadoop.log.dir=" + hadoopLogDir);
- vargs.add("-Dhadoop.root.logger=DEBUG,TLA");
- vargs.add("-Dhadoop.tasklog.taskid=" + task.getTaskID());
- vargs.add("-Dhadoop.tasklog.iscleanup=" + task.isTaskCleanupTask());
- vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
+ long logSize, String containerLogDir) {
+ vargs.add("-Dlog4j.configuration=container-log4j.properties");
+ vargs.add("-Dhadoop.yarn.mr.containerLogDir=" + containerLogDir);
+ vargs.add("-Dhadoop.yarn.mr.totalLogFileSize=" + logSize);
}
public static List<String> getVMCommand(
@@ -244,13 +227,12 @@ public class MapReduceChildJVM {
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
- setupLog4jProperties(vargs, logSize, logDir, task);
+ setupLog4jProperties(vargs, logSize, logDir);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
).isIncluded(task.getPartition())) {
- File prof = getTaskLogFile(logDir, attemptID, task.isTaskCleanupTask(),
- TaskLog.LogName.PROFILE);
+ File prof = getTaskLogFile(logDir, TaskLog.LogName.PROFILE);
vargs.add(String.format(conf.getProfileParams(), prof.toString()));
}
}
@@ -261,19 +243,11 @@ public class MapReduceChildJVM {
vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
vargs.add(attemptID.toString()); // pass task identifier
- // pass task log location
- // TODO: The following API uses system property hadoop.log.dir
- String attemptLogDir = getAttemptDir(logDir, attemptID, task.isTaskCleanupTask()).toString();
- vargs.add(attemptLogDir);
// Finally add the jvmID
vargs.add(String.valueOf(jvmID.getId()));
- vargs.add("1>"
- + getTaskLogFile(logDir, attemptID, task.isTaskCleanupTask(),
- TaskLog.LogName.STDERR));
- vargs.add("2>"
- + getTaskLogFile(logDir, attemptID, task.isTaskCleanupTask(),
- TaskLog.LogName.STDOUT));
+ vargs.add("1>" + getTaskLogFile(logDir, TaskLog.LogName.STDERR));
+ vargs.add("2>" + getTaskLogFile(logDir, TaskLog.LogName.STDOUT));
// Final commmand
StringBuilder mergedCommand = new StringBuilder();
@@ -281,8 +255,7 @@ public class MapReduceChildJVM {
mergedCommand.append(str).append(" ");
}
Vector<String> vargsFinal = new Vector<String>(8);
- vargsFinal.add("mkdir work; mkdir -p " + attemptLogDir + "; "
- + mergedCommand.toString());
+ vargsFinal.add("mkdir work;" + mergedCommand.toString());
return vargsFinal;
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Mon May 2 17:30:03 2011
@@ -18,14 +18,14 @@
package org.apache.hadoop.mapred;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
-import static java.util.concurrent.TimeUnit.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -42,19 +42,14 @@ import org.apache.hadoop.mapreduce.TaskT
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
-
-import org.apache.hadoop.mapred.YarnChild;
-
import org.apache.log4j.LogManager;
/**
@@ -77,8 +72,7 @@ class YarnChild {
int port = Integer.parseInt(args[1]);
final InetSocketAddress address = new InetSocketAddress(host, port);
final TaskAttemptID firstTaskid = TaskAttemptID.forName(args[2]);
- final String logLocation = args[3];
- int jvmIdInt = Integer.parseInt(args[4]);
+ int jvmIdInt = Integer.parseInt(args[3]);
JVMId jvmId = new JVMId(firstTaskid.getJobID(),
firstTaskid.getTaskType() == TaskType.MAP, jvmIdInt);
@@ -127,11 +121,7 @@ class YarnChild {
// Create the job-conf and set credentials
final JobConf job =
- configureTask(task, defaultConf.getCredentials(), jt, logLocation);
-
- //create the index file so that the log files
- //are viewable immediately
- TaskLog.syncLogs(logLocation, taskid, false);
+ configureTask(task, defaultConf.getCredentials(), jt);
// Initiate Java VM metrics
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
@@ -147,13 +137,9 @@ class YarnChild {
childUGI.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
- try {
- // use job-specified working directory
- FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
- taskFinal.run(job, umbilical); // run the task
- } finally {
- TaskLog.syncLogs(logLocation, taskid, false);
- }
+ // use job-specified working directory
+ FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
+ taskFinal.run(job, umbilical); // run the task
return null;
}
});
@@ -230,39 +216,6 @@ class YarnChild {
return jt;
}
- static void startTaskLogSync(final String logLocation) {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- try {
- if (taskid != null) {
- TaskLog.syncLogs(logLocation, taskid, false);
- }
- } catch (Throwable throwable) { }
- }
- });
- Thread t = new Thread() {
- public void run() {
- //every so often wake up and syncLogs so that we can track
- //logs of the currently running task
- while (true) {
- try {
- Thread.sleep(5000);
- if (taskid != null) {
- TaskLog.syncLogs(logLocation, taskid, false);
- }
- } catch (InterruptedException ie) {
- } catch (Throwable iee) {
- LOG.error("Error in syncLogs: " + iee);
- System.exit(-1);
- }
- }
- }
- };
- t.setName("syncLogs");
- t.setDaemon(true);
- t.start();
- }
-
static void configureLocalDirs(Task task, JobConf job) {
String[] localSysDirs = StringUtils.getTrimmedStrings(
System.getenv(YARNApplicationConstants.LOCAL_DIR_ENV));
@@ -271,7 +224,7 @@ class YarnChild {
}
static JobConf configureTask(Task task, Credentials credentials,
- Token<JobTokenIdentifier> jt, String logLocation) throws IOException {
+ Token<JobTokenIdentifier> jt) throws IOException {
final JobConf job = new JobConf(YARNApplicationConstants.JOB_CONF_FILE);
job.setCredentials(credentials);
// set tcp nodelay
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Mon May 2 17:30:03 2011
@@ -284,7 +284,7 @@ public class MRAppMaster extends Composi
// TODO:currently just wait for some time so clients can know the
// final states. Will be removed once RM come on.
try {
- Thread.sleep(15000);
+ Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Mon May 2 17:30:03 2011
@@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerTags;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -526,7 +527,8 @@ public abstract class TaskAttemptImpl im
container.setUser(conf.get(MRJobConfig.USER_NAME)); // TODO: Fix
File workDir = new File(ContainerBuilderHelper.getWorkDir());
- String logDir = new File(workDir, "logs").toString();
+ String containerLogDir =
+ new File("<" + ContainerTags.LOG_DIR + ">").toString();
String childTmpDir = new File(workDir, "tmp").toString();
String javaHome = "${JAVA_HOME}";
String nmLdLibraryPath =
@@ -541,12 +543,13 @@ public abstract class TaskAttemptImpl im
classPaths.add(workDir.toString()); // TODO
// Construct the actual Container
- container.addAllCommands(MapReduceChildJVM.getVMCommand(taskAttemptListener.getAddress(),
- remoteTask, javaHome, workDir.toString(), logDir,
- childTmpDir, jvmID));
-
- MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths, workDir.toString(),
- nmLdLibraryPath, remoteTask, localizedApplicationTokensFile);
+ container.addAllCommands(MapReduceChildJVM.getVMCommand(
+ taskAttemptListener.getAddress(), remoteTask, javaHome,
+ workDir.toString(), containerLogDir, childTmpDir, jvmID));
+
+ MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths,
+ workDir.toString(), containerLogDir, nmLdLibraryPath, remoteTask,
+ localizedApplicationTokensFile);
// Construct the actual Container
container.setContainerId(containerID);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Mon May 2 17:30:03 2011
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerTags;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -336,8 +337,8 @@ public class YARNRunner implements Clien
vargs.add(String.valueOf(applicationId.getClusterTimestamp()));
vargs.add(String.valueOf(applicationId.getId()));
vargs.add(YarnConfiguration.AM_FAIL_COUNT_STRING);
- vargs.add("1>logs/stdout");
- vargs.add("2>logs/stderr");
+ vargs.add("1><" + ContainerTags.LOG_DIR + ">/stdout");
+ vargs.add("2><" + ContainerTags.LOG_DIR + ">/stderr");
Vector<String> vargsFinal = new Vector<String>(8);
// Final commmand
@@ -345,7 +346,7 @@ public class YARNRunner implements Clien
for (CharSequence str : vargs) {
mergedCommand.append(str).append(" ");
}
- vargsFinal.add("mkdir logs;" + mergedCommand.toString());
+ vargsFinal.add(mergedCommand.toString());
LOG.info("Command to launch container for ApplicationMaster is : "
+ mergedCommand);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java Mon May 2 17:30:03 2011
@@ -22,8 +22,8 @@ public interface ContainerLaunchContext
public abstract String getEnv(String key);
public abstract List<String> getCommandList();
- public abstract String getCommand(int index);
- public abstract int getCommandCount();
+ public abstract String getCommand(int index); // TODO: Remove
+ public abstract int getCommandCount(); // TODO: Remove
public abstract void setContainerId(ContainerId containerId);
public abstract void setUser(String user);
@@ -48,6 +48,6 @@ public interface ContainerLaunchContext
public abstract void addAllCommands(List<String> commands);
public abstract void addCommand(String command);
- public abstract void removeCommand(int index);
+ public abstract void removeCommand(int index); // TODO: Remove
public abstract void clearCommands();
}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerTags.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerTags.java?rev=1098673&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerTags.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerTags.java Mon May 2 17:30:03 2011
@@ -0,0 +1,23 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.api.records;
+
+public enum ContainerTags {
+ LOG_DIR
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/container_manager.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/container_manager.proto?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/container_manager.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/container_manager.proto Mon May 2 17:30:03 2011
@@ -5,9 +5,9 @@ option java_generate_equals_and_hash = t
import "yarn_service_protos.proto";
-
-
-
+enum ContainerTagsProto {
+ LOG_DIR = 1;
+}
service ContainerManagerService {
rpc startContainer(StartContainerRequestProto) returns (StartContainerResponseProto);
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java?rev=1098673&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java Mon May 2 17:30:03 2011
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.Queue;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.spi.LoggingEvent;
+
+/**
+ * A simple log4j-appender for the task child's
+ * map-reduce system logs.
+ *
+ */
+@InterfaceStability.Unstable
+public class ContainerLogAppender extends FileAppender {
+ private String containerLogDir;
+ //so that log4j can configure it from the configuration(log4j.properties).
+ private int maxEvents;
+ private Queue<LoggingEvent> tail = null;
+ private boolean isCleanup;
+
+ @Override
+ public void activateOptions() {
+ synchronized (this) {
+ if (maxEvents > 0) {
+ tail = new LinkedList<LoggingEvent>();
+ }
+ setFile(new File(this.containerLogDir, "syslog").toString());
+ setAppend(true);
+ super.activateOptions();
+ }
+ }
+
+ @Override
+ public void append(LoggingEvent event) {
+ synchronized (this) {
+ if (tail == null) {
+ super.append(event);
+ } else {
+ if (tail.size() >= maxEvents) {
+ tail.remove();
+ }
+ tail.add(event);
+ }
+ }
+ }
+
+ public void flush() {
+ qw.flush();
+ }
+
+ @Override
+ public synchronized void close() {
+ if (tail != null) {
+ for(LoggingEvent event: tail) {
+ super.append(event);
+ }
+ }
+ super.close();
+ }
+
+ /**
+ * Getter/Setter methods for log4j.
+ */
+
+ public String getContainerLogDir() {
+ return this.containerLogDir;
+ }
+
+ public void setContainerLogDir(String containerLogDir) {
+ this.containerLogDir = containerLogDir;
+ }
+
+ private static final int EVENT_SIZE = 100;
+
+ public long getTotalLogFileSize() {
+ return maxEvents * EVENT_SIZE;
+ }
+
+ public void setTotalLogFileSize(long logSize) {
+ maxEvents = (int) logSize / EVENT_SIZE;
+ }
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c Mon May 2 17:30:03 2011
@@ -271,7 +271,7 @@ char* get_job_log_directory(const char*
fprintf(LOGFILE, "Log directory %s is not configured.\n", TT_LOG_DIR_KEY);
return NULL;
}
- char *result = concatenate("%s/userlogs/%s", "job log dir", 2, log_dir,
+ char *result = concatenate("%s/%s", "job log dir", 2, log_dir,
jobid);
if (result == NULL) {
fprintf(LOGFILE, "failed to get memory in get_job_log_directory for %s"
@@ -648,7 +648,7 @@ int initialize_user(const char *user) {
* Function to prepare the job directories for the task JVM.
*/
int initialize_job(const char *user, const char *jobid,
- const char* credentials, char* const* args) {
+ const char* nmPrivate_credentials_file, char* const* args) {
if (jobid == NULL || user == NULL) {
fprintf(LOGFILE, "Either jobid is null or the user passed is null.\n");
return INVALID_ARGUMENT_NUMBER;
@@ -672,7 +672,7 @@ int initialize_job(const char *user, con
}
// open up the credentials file
- int cred_file = open_file_as_task_tracker(credentials);
+ int cred_file = open_file_as_task_tracker(nmPrivate_credentials_file);
if (cred_file == -1) {
return -1;
}
@@ -710,15 +710,21 @@ int initialize_job(const char *user, con
return -1;
}
+ char *nmPrivate_credentials_file_copy = strdup(nmPrivate_credentials_file);
char *cred_file_name = concatenate("%s/%s", "cred file", 2,
- primary_job_dir, CREDENTIALS_FILENAME);
+ primary_job_dir, basename(nmPrivate_credentials_file_copy));
if (cred_file_name == NULL) {
+ free(nmPrivate_credentials_file_copy);
return -1;
}
- if (copy_file(cred_file, credentials, cred_file_name, S_IRUSR|S_IWUSR) != 0){
+ if (copy_file(cred_file, nmPrivate_credentials_file,
+ cred_file_name, S_IRUSR|S_IWUSR) != 0){
+ free(nmPrivate_credentials_file_copy);
return -1;
}
+ free(nmPrivate_credentials_file_copy);
+
fclose(stdin);
fflush(LOGFILE);
if (LOGFILE != stdout) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Mon May 2 17:30:03 2011
@@ -84,8 +84,8 @@ public abstract class ContainerExecutor
* @param launchCtxt
*/
public abstract int launchContainer(Container container, Path nmLocal,
- String user, String appId, List<Path> appDirs, String stdout,
- String stderr) throws IOException;
+ String user, String appId, Path appLogDir, List<Path> appDirs)
+ throws IOException;
public abstract boolean signalContainer(String user, String pid,
Signal signal)
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Mon May 2 17:30:03 2011
@@ -81,27 +81,28 @@ public class DefaultContainerExecutor ex
}
@Override
- public int launchContainer(Container container, Path nmLocal,
- String user, String appId, List<Path> appDirs, String stdout,
- String stderr) throws IOException {
+ public int launchContainer(Container container, Path nmLocal, String user,
+ String appId, Path appLogDir, List<Path> appDirs) throws IOException {
// create container dirs
+ String containerIdStr = ConverterUtils.toString(container.getContainerID());
for (Path p : appDirs) {
- lfs.mkdir(new Path(p,
- ConverterUtils.toString(container.getContainerID())),
- null, true);
+ lfs.mkdir(new Path(p, containerIdStr), null, false);
}
+ lfs.mkdir(new Path(appLogDir, containerIdStr), null, false);
// copy launch script to work dir
- Path appWorkDir = new Path(appDirs.get(0), container.toString());
+ // TODO: ROUND_ROBIN Below
+ Path appWorkDir = new Path(appDirs.get(0), containerIdStr);
Path launchScript = new Path(nmLocal, ContainerLaunch.CONTAINER_SCRIPT);
Path launchDst = new Path(appWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
lfs.util().copy(launchScript, launchDst);
// copy container tokens to work dir
Path appTokens = new Path(nmLocal, String.format(
ContainerLocalizer.TOKEN_FILE_FMT,
- ConverterUtils.toString(container.getContainerID())));
+ containerIdStr));
Path tokenDst =
new Path(appWorkDir, ContainerLaunch.CONTAINER_TOKENS);
+
lfs.util().copy(appTokens, tokenDst);
// create log dir under app
// fork script
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Mon May 2 17:30:03 2011
@@ -147,12 +147,11 @@ public class LinuxContainerExecutor exte
}
@Override
- public int launchContainer(Container container, Path nmLocal,
- String user, String appId, List<Path> appDirs, String stdout,
- String stderr) throws IOException {
- Path appWorkDir = new Path(appDirs.get(0), container.toString());
+ public int launchContainer(Container container, Path nmLocal, String user,
+ String appId, Path appLogDir, List<Path> appDirs) throws IOException {
+ Path appWorkDir = new Path(appDirs.get(0), container.toString()); // TODO: Use ROUND_ROBIN
Path launchScript = new Path(nmLocal, ContainerLaunch.CONTAINER_SCRIPT);
- Path appToken = new Path(nmLocal, String.format(
+ Path nmPrivateAppTokenFile = new Path(nmLocal, String.format(
ContainerLocalizer.TOKEN_FILE_FMT,
ConverterUtils.toString(container.getContainerID())));
List<String> command = new ArrayList<String>(
@@ -163,7 +162,7 @@ public class LinuxContainerExecutor exte
container.toString(),
appWorkDir.toString(),
launchScript.toUri().getPath().toString(),
- appToken.toUri().getPath().toString()));
+ nmPrivateAppTokenFile.toUri().getPath().toString()));
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
launchCommandObjs.put(container.getLaunchContext().getContainerId(), shExec);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java Mon May 2 17:30:03 2011
@@ -51,7 +51,7 @@ public class NMConfig {
public static final String DEFAULT_NM_LOCAL_DIR = "/tmp/nm-local-dir";
- public static final String NM_LOG_DIR = NM_PREFIX + "log.dir";
+ public static final String NM_LOG_DIR = NM_PREFIX + "log.dir"; // TODO: Rename
public static final String DEFAULT_NM_LOG_DIR = "/tmp/logs";
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Mon May 2 17:30:03 2011
@@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.Creat
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import java.io.DataOutputStream;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -58,13 +59,16 @@ public class ContainerLaunch implements
private final Application app;
private final Container container;
private final Path sysDir;
+ private final Path appLogDir;
private final List<Path> appDirs;
public ContainerLaunch(Dispatcher dispatcher, ContainerExecutor exec,
- Application app, Container container, Path sysDir, List<Path> appDirs) {
+ Application app, Container container, Path sysDir, Path appLogDir,
+ List<Path> appDirs) {
this.app = app;
this.exec = exec;
this.sysDir = sysDir;
+ this.appLogDir = appLogDir;
this.appDirs = appDirs;
this.container = container;
this.dispatcher = dispatcher;
@@ -79,16 +83,28 @@ public class ContainerLaunch implements
final Map<String,String> env = launchContext.getAllEnv();
final List<String> command = launchContext.getCommandList();
int ret = -1;
+
+ // Variable expansion
+ // Before the container script gets written out.
+ List<String> cmds = container.getLaunchContext().getCommandList();
+ List<String> newCmds = new ArrayList<String>(cmds.size());
+ String containerIdStr = ConverterUtils.toString(container.getContainerID());
+ Path containerLogDir = new Path(appLogDir, containerIdStr);
+ for (String str : cmds) {
+ newCmds.add(str.replace("<LOG_DIR>", containerLogDir.toUri()
+ .getPath()));
+ }
+ container.getLaunchContext().clearCommands();
+ container.getLaunchContext().addAllCommands(newCmds);
+
try {
FileContext lfs = FileContext.getLocalFSFileContext();
- Path launchSysDir =
- new Path(sysDir, ConverterUtils.toString(container.getContainerID()));
+ Path launchSysDir = new Path(sysDir, containerIdStr);
lfs.mkdir(launchSysDir, null, true);
Path launchPath = new Path(launchSysDir, CONTAINER_SCRIPT);
Path tokensPath =
- new Path(launchSysDir, String.format(
- ContainerLocalizer.TOKEN_FILE_FMT,
- ConverterUtils.toString(container.getContainerID())));
+ new Path(launchSysDir, String.format(
+ ContainerLocalizer.TOKEN_FILE_FMT, containerIdStr));
DataOutputStream launchOut = null;
DataOutputStream tokensOut = null;
@@ -109,9 +125,10 @@ public class ContainerLaunch implements
dispatcher.getEventHandler().handle(new ContainerEvent(
container.getContainerID(),
ContainerEventType.CONTAINER_LAUNCHED));
+
ret =
exec.launchContainer(container, launchSysDir, user, app.toString(),
- appDirs, null, null);
+ appLogDir, appDirs);
if (ret == ExitCode.KILLED.getExitCode()) {
// If the process was killed, Send container_cleanedup_after_kill and
// just break out of this method.
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Mon May 2 17:30:03 2011
@@ -136,20 +136,25 @@ public class ContainersLauncher extends
case LAUNCH_CONTAINER:
Application app =
context.getApplications().get(containerId.getAppId());
+ String appIdStr = ConverterUtils.toString(app.getAppId());
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
for (Path p : localDirs) {
Path usersdir = new Path(p, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, userName);
Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
- appDirs.add(new Path(appsdir, app.toString()));
+ appDirs.add(new Path(appsdir, appIdStr));
}
Path appSysDir =
- new Path(sysDirs.get(0), ConverterUtils.toString(app.getAppId()));
+ new Path(sysDirs.get(0), appIdStr);
+ Path appLogDir =
+ new Path(logDirs.get(0), appIdStr);
+ // TODO: ROUND_ROBIN above.
// TODO set in Application
//Path appLogDir = new Path(logDirs.get(0), app.toString());
ContainerLaunch launch =
new ContainerLaunch(dispatcher, exec, app,
- event.getContainer(), appSysDir, appDirs);
+ event.getContainer(), appSysDir, appLogDir, appDirs);
+ // TODO: ROUND_ROBIN above.
running.put(containerId,
new RunningContainer(userName,
containerLauncher.submit(launch)));
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Mon May 2 17:30:03 2011
@@ -368,7 +368,8 @@ public class ContainerLocalizer {
}
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appCacheDirs);
conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), userCacheDirs);
- lfs.mkdir(logDir, null, true);
+ Path appLogDir = new Path(logDir, appId);
+ lfs.mkdir(appLogDir, null, true);
}
public static void writeLaunchEnv(OutputStream out,
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Mon May 2 17:30:03 2011
@@ -147,7 +147,7 @@ public class LocalizedResource implement
public void transition(LocalizedResource rsrc, ResourceEvent event) {
ResourceRequestEvent req = (ResourceRequestEvent) event;
LocalizerContext ctxt = req.getContext();
- ContainerId container = ctxt.getContainer();
+ ContainerId container = ctxt.getContainerId();
rsrc.ref.add(container);
rsrc.dispatcher.getEventHandler().handle(
new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt));
@@ -190,7 +190,7 @@ public class LocalizedResource implement
public void transition(LocalizedResource rsrc, ResourceEvent event) {
// notify waiting containers
ResourceRequestEvent reqEvent = (ResourceRequestEvent) event;
- ContainerId container = reqEvent.getContext().getContainer();
+ ContainerId container = reqEvent.getContext().getContainerId();
rsrc.ref.add(container);
rsrc.dispatcher.getEventHandler().handle(
new ContainerResourceLocalizedEvent(
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java Mon May 2 17:30:03 2011
@@ -1,3 +1,21 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import org.apache.hadoop.security.Credentials;
@@ -6,13 +24,13 @@ import org.apache.hadoop.yarn.api.record
public class LocalizerContext {
private final String user;
- private final ContainerId container;
+ private final ContainerId containerId;
private final Credentials credentials;
- public LocalizerContext(String user, ContainerId container,
+ public LocalizerContext(String user, ContainerId containerId,
Credentials credentials) {
this.user = user;
- this.container = container;
+ this.containerId = containerId;
this.credentials = credentials;
}
@@ -20,8 +38,8 @@ public class LocalizerContext {
return user;
}
- public ContainerId getContainer() {
- return container;
+ public ContainerId getContainerId() {
+ return containerId;
}
public Credentials getCredentials() {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Mon May 2 17:30:03 2011
@@ -373,6 +373,7 @@ public class ResourceLocalizationService
(LocalizerResourceRequestEvent)event;
if (null == localizer) {
LOG.info("Created localizer for " + req.getLocalizerId());
+ // TODO: ROUND_ROBIN below.
localizer = new LocalizerRunner(req.getContext(),
sysDirs.get(0), req.getLocalizerId(), logDirs.get(0));
trackers.put(locId, localizer);
@@ -399,7 +400,7 @@ public class ResourceLocalizationService
final LocalizerContext context;
final String localizerId;
final Path nmPrivate;
- final Path logDir;
+ final Path rootLogDir;
final Map<LocalResourceRequest,LocalizerResourceRequestEvent> scheduled;
final List<LocalizerResourceRequestEvent> pending;
@@ -420,7 +421,7 @@ public class ResourceLocalizationService
this.nmPrivate = nmPrivate;
this.context = context;
this.localizerId = localizerId;
- this.logDir = logDir;
+ this.rootLogDir = logDir;
this.pending = pending;
this.scheduled = scheduled;
}
@@ -514,14 +515,14 @@ public class ResourceLocalizationService
assoc.getResource().unlock();
response.setLocalizerAction(LocalizerAction.DIE);
dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(context.getContainer(),
+ new ContainerResourceFailedEvent(context.getContainerId(),
req, stat.getException()));
break;
default:
LOG.info("Unknown status: " + stat.getStatus());
response.setLocalizerAction(LocalizerAction.DIE);
dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(context.getContainer(),
+ new ContainerResourceFailedEvent(context.getContainerId(),
req, stat.getException()));
break;
}
@@ -542,8 +543,8 @@ public class ResourceLocalizationService
ContainerLocalizer.TOKEN_FILE_FMT, localizerId));
FileContext lfs = getLocalFileContext(getConfig());
tokenOut = lfs.create(cTokens, EnumSet.of(CREATE, OVERWRITE));
- LOG.info("Writing credentials to " + cTokens.toString() +
- ". Credentials list: ");
+ LOG.info("Writing credentials to the nmPrivate file "
+ + cTokens.toString() + ". Credentials list: ");
for (Token<? extends TokenIdentifier> tk :
credentials.getAllTokens()) {
LOG.info(tk.getService() + " : " + tk.encodeToUrlString());
@@ -556,8 +557,8 @@ public class ResourceLocalizationService
}
// 2) exec initApplication and wait
exec.startLocalizer(nmPrivate, locAddr, context.getUser(),
- ConverterUtils.toString(context.getContainer().getAppId()),
- localizerId, logDir, localDirs);
+ ConverterUtils.toString(context.getContainerId().getAppId()),
+ localizerId, rootLogDir, localDirs);
} catch (Exception e) {
// 3) on error, report failure to Container and signal ABORT
// 3.1) notify resource of failed localization
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java Mon May 2 17:30:03 2011
@@ -31,7 +31,7 @@ public class LocalizerResourceRequestEve
public LocalizerResourceRequestEvent(LocalizedResource resource,
LocalResourceVisibility vis, LocalizerContext context) {
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
- ConverterUtils.toString(context.getContainer()));
+ ConverterUtils.toString(context.getContainerId()));
this.vis = vis;
this.context = context;
this.resource = resource;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Mon May 2 17:30:03 2011
@@ -18,8 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOCAL_DIR;
-import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOCAL_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOG_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOG_DIR;
import java.io.File;
import java.io.FileReader;
@@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
@@ -84,7 +83,7 @@ public class ContainerLogsPage extends N
if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
// TODO: Get the following from logs' owning component.
File containerLogsDir =
- getContainerLogDir(this.conf, this.nmContext, containerId);
+ getContainerLogDir(this.conf, containerId);
File logFile = new File(containerLogsDir, $(CONTAINER_LOG_TYPE));
div.h1(logFile.getName());
long start =
@@ -132,7 +131,7 @@ public class ContainerLogsPage extends N
} else {
// Just print out the log-types
File containerLogsDir =
- getContainerLogDir(this.conf, this.nmContext, containerId);
+ getContainerLogDir(this.conf, containerId);
// TODO: No nested dir structure. Fix MR userlogs.
for (File logFile : containerLogsDir.listFiles()) {
div
@@ -152,23 +151,16 @@ public class ContainerLogsPage extends N
}
}
- static File getContainerLogDir(Configuration conf,
- Context nmContext, ContainerId containerId) {
- String[] sLocalDirs =
- conf.getStrings(NM_LOCAL_DIR, DEFAULT_NM_LOCAL_DIR);
- File localDir = new File(sLocalDirs[0]); // TODO: Fix
- File userCacheDir =
- new File(localDir, ContainerLocalizer.USERCACHE);
- String user =
- nmContext.getContainers().get(containerId).getUser();
- File userDir = new File(userCacheDir, user);
- File appCacheDir = new File(userDir, ContainerLocalizer.APPCACHE);
- File containerDir =
- new File(new File(appCacheDir,
- ConverterUtils.toString(containerId
- .getAppId())), ConverterUtils.toString(containerId));
- File containerLogsDir = new File(containerDir, "logs");
- return containerLogsDir;
+ static File
+ getContainerLogDir(Configuration conf, ContainerId containerId) {
+ String[] logDirs =
+ conf.getStrings(NM_LOG_DIR, DEFAULT_NM_LOG_DIR);
+ File logDir = new File(logDirs[0]); // TODO: In case of ROUND_ROBIN
+ String appIdStr = ConverterUtils.toString(containerId.getAppId());
+ File appLogDir = new File(logDir, appIdStr);
+ String containerIdStr = ConverterUtils.toString(containerId);
+ File containerLogDir = new File(appLogDir, containerIdStr);
+ return containerLogDir;
}
}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/container-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/container-log4j.properties?rev=1098673&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/container-log4j.properties (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/resources/container-log4j.properties Mon May 2 17:30:03 2011
@@ -0,0 +1,23 @@
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=DEBUG,CLA
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# ContainerLog Appender
+#
+
+#Default values
+hadoop.yarn.mr.containerLogDir=null
+hadoop.yarn.mr.totalLogFileSize=100
+
+log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender
+log4j.appender.CLA.containerLogDir=${hadoop.yarn.mr.containerLogDir}
+log4j.appender.CLA.totalLogFileSize=${hadoop.yarn.mr.totalLogFileSize}
+
+log4j.appender.CLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java Mon May 2 17:30:03 2011
@@ -17,10 +17,24 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
-import java.io.IOException;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.InetSocketAddress;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -51,13 +65,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
-
import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.mockito.ArgumentMatcher;
-import static org.mockito.Matchers.*;
-import static org.mockito.Mockito.*;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -172,7 +181,8 @@ public class TestContainerLocalizer {
verify(spylfs).open(tokenPath);
// verify log dir creation
- verify(spylfs).mkdir(eq(logDir), isA(FsPermission.class), anyBoolean());
+ verify(spylfs).mkdir(eq(new Path(logDir, appId)),
+ isA(FsPermission.class), anyBoolean());
// verify downloaded resources reported to NM
verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA)));
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java Mon May 2 17:30:03 2011
@@ -219,7 +219,7 @@ public class TestLocalizedResource {
public boolean matches(Object o) {
if (!(o instanceof LocalizerResourceRequestEvent)) return false;
LocalizerResourceRequestEvent evt = (LocalizerResourceRequestEvent) o;
- return idRef == evt.getContext().getContainer()
+ return idRef == evt.getContext().getContainerId()
&& type == evt.getType()
&& vis == evt.getVisibility()
&& creds == evt.getContext().getCredentials();
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Mon May 2 17:30:03 2011
@@ -123,7 +123,7 @@ public class TestNMWebServer {
// ContainerLogDir should be created
File containerLogDir =
ContainerLogsPage.ContainersLogsBlock.getContainerLogDir(conf,
- nmContext, containerId);
+ containerId);
containerLogDir.mkdirs();
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
Writer writer = new FileWriter(new File(containerLogDir, fileType));
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Mon May 2 17:30:03 2011
@@ -39,12 +39,14 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+@Ignore
public class TestCapacityScheduler extends TestCase {
private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java?rev=1098673&r1=1098672&r2=1098673&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java Mon May 2 17:30:03 2011
@@ -73,8 +73,10 @@ import org.apache.hadoop.yarn.security.C
import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Ignore;
import org.junit.Test;
+@Ignore
public class TestContainerTokenSecretManager {
private static Log LOG = LogFactory