You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2009/10/20 09:23:30 UTC
svn commit: r826980 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/contrib/streaming/src/test/org/apache/hadoop/streaming/
src/java/org/apache/hadoop/mapred/
Author: yhemanth
Date: Tue Oct 20 07:23:30 2009
New Revision: 826980
URL: http://svn.apache.org/viewvc?rev=826980&view=rev
Log:
Merge -r 826978:826979 from trunk to branch-0.21 to fix MAPREDUCE-1086.
Added:
hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
- copied unchanged from r826979, hadoop/mapreduce/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskLog.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskRunner.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=826980&r1=826979&r2=826980&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Tue Oct 20 07:23:30 2009
@@ -774,3 +774,6 @@
MAPREDUCE-1111. JT Jetty UI not working if we run mumak.sh off
packaged distribution directory. (hong tang via mahadev)
+ MAPREDUCE-1086. Setup Hadoop logging environment for tasks to point to
+ task related parameters. (Ravi Gummadi via yhemanth)
+
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskLog.java?rev=826980&r1=826979&r2=826980&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskLog.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskLog.java Tue Oct 20 07:23:30 2009
@@ -59,13 +59,9 @@
private static final File LOG_DIR =
new File(getBaseLogDir(), USERLOGS_DIR_NAME).getAbsoluteFile();
+ // localFS is set in (and used by) writeToIndexFile()
static LocalFileSystem localFS = null;
static {
- try {
- localFS = FileSystem.getLocal(new Configuration());
- } catch (IOException ioe) {
- LOG.warn("Getting local file system failed.");
- }
if (!LOG_DIR.exists()) {
boolean b = LOG_DIR.mkdirs();
if (!b) {
@@ -200,6 +196,10 @@
File indexFile = getIndexFile(currentTaskid.toString(), isCleanup);
Path indexFilePath = new Path(indexFile.getAbsolutePath());
Path tmpIndexFilePath = new Path(tmpIndexFile.getAbsolutePath());
+
+ if (localFS == null) {// set localFS once
+ localFS = FileSystem.getLocal(new Configuration());
+ }
localFS.rename (tmpIndexFilePath, indexFilePath);
}
private static void resetPrevLengths(TaskAttemptID firstTaskid) {
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=826980&r1=826979&r2=826980&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskRunner.java Tue Oct 20 07:23:30 2009
@@ -210,7 +210,8 @@
stderr);
Map<String, String> env = new HashMap<String, String>();
- errorInfo = getVMEnvironment(errorInfo, workDir, conf, env);
+ errorInfo = getVMEnvironment(errorInfo, workDir, conf, env,
+ taskid, logSize);
jvmManager.launchJvm(this,
jvmManager.constructJvmEnv(setup,vargs,stdout,stderr,logSize,
@@ -479,6 +480,7 @@
}
/**
+ * sets the environment variables needed for task jvm and its children.
* @param errorInfo
* @param workDir
* @param env
@@ -486,7 +488,7 @@
* @throws Throwable
*/
private String getVMEnvironment(String errorInfo, File workDir, JobConf conf,
- Map<String, String> env)
+ Map<String, String> env, TaskAttemptID taskid, long logSize)
throws Throwable {
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
@@ -498,6 +500,18 @@
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
+ // for the child of task jvm, set hadoop.root.logger
+ env.put("HADOOP_ROOT_LOGGER", "INFO,TLA");
+ String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
+ if (hadoopClientOpts == null) {
+ hadoopClientOpts = "";
+ } else {
+ hadoopClientOpts = hadoopClientOpts + " ";
+ }
+ hadoopClientOpts = hadoopClientOpts + "-Dhadoop.tasklog.taskid=" + taskid
+ + " -Dhadoop.tasklog.totalLogFileSize=" + logSize;
+ env.put("HADOOP_CLIENT_OPTS", "\"" + hadoopClientOpts + "\"");
+
// add the env variables passed by the user
String mapredChildEnv = getChildEnv(conf);
if (mapredChildEnv != null && mapredChildEnv.length() > 0) {