You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2013/07/23 17:32:34 UTC
svn commit: r1506114 - in /hadoop/common/branches/branch-1: ./ src/mapred/
src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapreduce/
Author: acmurthy
Date: Tue Jul 23 15:32:33 2013
New Revision: 1506114
URL: http://svn.apache.org/r1506114
Log:
MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. Contributed by Arun C. Murthy.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Jul 23 15:32:33 2013
@@ -27,6 +27,8 @@ Release 1.3.0 - unreleased
MAPREDUCE-5406. Improve logging around Task Tracker exiting with JVM manager
inconsistent state. (Chelsey Chang via cnauroth)
+ MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. (acmurthy)
+
BUG FIXES
MAPREDUCE-5047. keep.failed.task.files=true causes job failure on
Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Tue Jul 23 15:32:33 2013
@@ -589,6 +589,22 @@
</property>
<property>
+ <name>mapred.map.child.log.level</name>
+ <value>INFO</value>
+ <description>The logging level for the map task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+</property>
+
+<property>
+ <name>mapred.reduce.child.log.level</name>
+ <value>INFO</value>
+ <description>The logging level for the reduce task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+</property>
+
+<property>
<name>mapred.inmem.merge.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java Tue Jul 23 15:32:33 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
+import org.apache.log4j.Level;
/**
* A map/reduce job configuration.
@@ -337,6 +338,29 @@ public class JobConf extends Configurati
private Credentials credentials = new Credentials();
/**
+ * Configuration key to set the logging {@link Level} for the map task.
+ *
+ * The allowed logging levels are:
+ * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ */
+ public static final String MAPRED_MAP_TASK_LOG_LEVEL =
+ "mapred.map.child.log.level";
+
+ /**
+ * Configuration key to set the logging {@link Level} for the reduce task.
+ *
+ * The allowed logging levels are:
+ * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ */
+ public static final String MAPRED_REDUCE_TASK_LOG_LEVEL =
+ "mapred.reduce.child.log.level";
+
+ /**
+ * Default logging level for map/reduce tasks.
+ */
+ public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
+
+ /**
* Construct a map/reduce job configuration.
*/
public JobConf() {
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java Tue Jul 23 15:32:33 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.*;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
/** Runs a map task. */
class MapTaskRunner extends TaskRunner {
@@ -69,4 +70,11 @@ class MapTaskRunner extends TaskRunner {
return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, super.getChildEnv(jobConf));
}
+ @Override
+ public Level getLogLevel(JobConf jobConf) {
+ return Level.toLevel(jobConf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL,
+ JobConf.DEFAULT_LOG_LEVEL.toString()),
+ Level.INFO);
+ }
+
}
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java Tue Jul 23 15:32:33 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.*;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
/** Runs a reduce task. */
class ReduceTaskRunner extends TaskRunner {
@@ -73,4 +74,11 @@ class ReduceTaskRunner extends TaskRunne
super.getChildEnv(jobConf));
}
+ @Override
+ public Level getLogLevel(JobConf jobConf) {
+ return Level.toLevel(jobConf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL,
+ JobConf.DEFAULT_LOG_LEVEL.toString()),
+ Level.INFO);
+ }
+
}
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Tue Jul 23 15:32:33 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
/** Base class that runs a task in a separate process. Tasks are run in a
* separate process in order to isolate the map/reduce system code from bugs in
@@ -183,6 +184,13 @@ abstract class TaskRunner extends Thread
return jobConf.get(JobConf.MAPRED_TASK_ENV);
}
+ /**
+ * Get the log {@link Level} for the child map/reduce tasks.
+ * @param jobConf
+ * @return the log-level for the child map/reduce tasks
+ */
+ public abstract Level getLogLevel(JobConf jobConf);
+
@Override
public final void run() {
String errorInfo = "Child Error";
@@ -477,7 +485,7 @@ abstract class TaskRunner extends Thread
vargs.add("-Dlog4j.configuration=task-log4j.properties");
vargs.add("-Dhadoop.log.dir=" +
new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
- vargs.add("-Dhadoop.root.logger=INFO,TLA");
+ vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA");
vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask());
vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java Tue Jul 23 15:32:33 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.HadoopTestCase;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
public class TestChild extends HadoopTestCase {
private static String TEST_ROOT_DIR =
@@ -68,6 +69,12 @@ public class TestChild extends HadoopTes
mapJavaOpts,
mapJavaOpts, MAP_OPTS_VAL);
}
+
+ Level logLevel =
+ Level.toLevel(conf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL,
+ Level.INFO.toString()));
+ assertEquals(JobConf.MAPRED_MAP_TASK_LOG_LEVEL + "has value of " +
+ logLevel, logLevel, Level.OFF);
}
}
@@ -94,6 +101,12 @@ public class TestChild extends HadoopTes
reduceJavaOpts,
reduceJavaOpts, REDUCE_OPTS_VAL);
}
+
+ Level logLevel =
+ Level.toLevel(conf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL,
+ Level.INFO.toString()));
+ assertEquals(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL + "has value of " +
+ logLevel, logLevel, Level.OFF);
}
}
@@ -108,6 +121,9 @@ public class TestChild extends HadoopTes
conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, REDUCE_OPTS_VAL);
}
+ conf.set(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, Level.OFF.toString());
+ conf.set(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, Level.OFF.toString());
+
Job job = MapReduceTestUtil.createJob(conf, inDir, outDir,
numMaps, numReds);
job.setMapperClass(MyMapper.class);