You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sa...@apache.org on 2014/01/17 18:53:23 UTC
svn commit: r1559202 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/...
Author: sandy
Date: Fri Jan 17 17:53:23 2014
New Revision: 1559202
URL: http://svn.apache.org/r1559202
Log:
MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params is specified (Gera Shegalov via Sandy Ryza)
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
- copied unchanged from r1559201, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1559202&r1=1559201&r2=1559202&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Jan 17 17:53:23 2014
@@ -188,6 +188,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
(Chuan Liu via cnauroth)
+ MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params
+ is specified (Gera Shegalov via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1559202&r1=1559201&r2=1559202&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Fri Jan 17 17:53:23 2014
@@ -212,19 +212,11 @@ public class MapReduceChildJVM {
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
).isIncluded(task.getPartition())) {
- vargs.add(
- String.format(
- conf.getProfileParams(),
- getTaskLogFile(TaskLog.LogName.PROFILE)
- )
- );
- if (task.isMapTask()) {
- vargs.add(conf.get(MRJobConfig.TASK_MAP_PROFILE_PARAMS, ""));
- }
- else {
- vargs.add(conf.get(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, ""));
- }
-
+ final String profileParams = conf.get(task.isMapTask()
+ ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
+ : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
+ vargs.add(String.format(profileParams,
+ getTaskLogFile(TaskLog.LogName.PROFILE)));
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1559202&r1=1559201&r2=1559202&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Fri Jan 17 17:53:23 2014
@@ -978,6 +978,31 @@
</property>
<property>
+ <name>mapreduce.task.profile.params</name>
+ <value></value>
+ <description>JVM profiler parameters used to profile map and reduce task
+ attempts. This string may contain a single format specifier %s that will
+ be replaced by the path to profile.out in the task attempt log directory.
+ To specify different profiling options for map tasks and reduce tasks,
+ more specific parameters mapreduce.task.profile.map.params and
+ mapreduce.task.profile.reduce.params should be used.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.map.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Map-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.reduce.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Reduce-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
<name>mapreduce.task.skip.start.attempts</name>
<value>2</value>
<description> The number of Task attempts AFTER which skip mode