You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/10/18 23:45:39 UTC
svn commit: r1185887 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hado...
Author: acmurthy
Date: Tue Oct 18 21:45:38 2011
New Revision: 1185887
URL: http://svn.apache.org/viewvc?rev=1185887&view=rev
Log:
MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and tasks. Contributed by Todd Lipcon.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1185887&r1=1185886&r2=1185887&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Oct 18 21:45:38 2011
@@ -1669,6 +1669,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2762. Cleanup MR staging directory on completion. (mahadev via
acmurthy)
+ MAPREDUCE-3165. Ensure logging options are set correctly for MR AM and
+ tasks. (todd via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1185887&r1=1185886&r2=1185887&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Tue Oct 18 21:45:38 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskLog.LogName;
import org.apache.hadoop.mapreduce.ID;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -93,7 +94,9 @@ public class MapReduceChildJVM {
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV)
);
- // Set logging level
+ // Set logging level in the environment.
+ // This is so that, if the child forks another "bin/hadoop" (common in
+ // streaming) it will have the correct loglevel.
environment.put(
"HADOOP_ROOT_LOGGER",
getChildLogLevel(conf, task.isMapTask()) + ",CLA");
@@ -110,7 +113,7 @@ public class MapReduceChildJVM {
// properties.
long logSize = TaskLog.getTaskLogLength(conf);
Vector<String> logProps = new Vector<String>(4);
- setupLog4jProperties(logProps, logSize);
+ setupLog4jProperties(task, logProps, logSize);
Iterator<String> it = logProps.iterator();
StringBuffer buffer = new StringBuffer();
while (it.hasNext()) {
@@ -163,11 +166,11 @@ public class MapReduceChildJVM {
return adminClasspath + " " + userClasspath;
}
- private static void setupLog4jProperties(Vector<String> vargs,
+ private static void setupLog4jProperties(Task task,
+ Vector<String> vargs,
long logSize) {
- vargs.add("-Dlog4j.configuration=container-log4j.properties");
- vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
+ String logLevel = getChildLogLevel(task.conf, task.isMapTask());
+ MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
}
public static List<String> getVMCommand(
@@ -222,7 +225,7 @@ public class MapReduceChildJVM {
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
- setupLog4jProperties(vargs, logSize);
+ setupLog4jProperties(task, vargs, logSize);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1185887&r1=1185886&r2=1185887&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Tue Oct 18 21:45:38 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -340,4 +341,19 @@ public class MRApps extends Apps {
}
return result;
}
+
+ /**
+ * Add the JVM system properties necessary to configure {@link ContainerLogAppender}.
+ * @param logLevel the desired log level (eg INFO/WARN/DEBUG)
+ * @param logSize See {@link ContainerLogAppender#setTotalLogFileSize(long)}
+ * @param vargs the argument list to append to
+ */
+ public static void addLog4jSystemProperties(
+ String logLevel, long logSize, List<String> vargs) {
+ vargs.add("-Dlog4j.configuration=container-log4j.properties");
+ vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
+ vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1185887&r1=1185886&r2=1185887&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Oct 18 21:45:38 2011
@@ -326,9 +326,9 @@ public interface MRJobConfig {
public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1536m";
/** Root Logging level passed to the MR app master.*/
- public static final String MR_AM_LOG_OPTS =
- MR_AM_PREFIX+"log-opts";
- public static final String DEFAULT_MR_AM_LOG_OPTS = "INFO";
+ public static final String MR_AM_LOG_LEVEL =
+ MR_AM_PREFIX+"log.level";
+ public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
/**The number of splits when reporting progress in MR*/
public static final String MR_AM_NUM_PROGRESS_SPLITS =
@@ -439,8 +439,9 @@ public interface MRJobConfig {
public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
+ // Environment variables used by Pipes. (TODO: these
+ // do not appear to be used by current pipes source code!)
public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
-
public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
// This should be the directory where splits file gets localized on the node
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1185887&r1=1185886&r2=1185887&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Tue Oct 18 21:45:38 2011
@@ -20,7 +20,9 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Vector;
@@ -320,14 +322,14 @@ public class YARNRunner implements Clien
}
// Setup the command to run the AM
- Vector<CharSequence> vargs = new Vector<CharSequence>(8);
+ List<String> vargs = new ArrayList<String>(8);
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+ // TODO: why do we use 'conf' some places and 'jobConf' others?
long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
- vargs.add("-Dlog4j.configuration=container-log4j.properties");
- vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
- + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
+ String logLevel = jobConf.get(
+ MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
+ MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));