You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 06:52:59 UTC
svn commit: r1079183 - in
/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop:
mapred/MapTask.java mapreduce/server/tasktracker/TTConfig.java
Author: omalley
Date: Tue Mar 8 05:52:59 2011
New Revision: 1079183
URL: http://svn.apache.org/viewvc?rev=1079183&view=rev
Log:
commit db221627d41f6cb801bbf25da25fa6d822937d28
Author: Mahadev Konar <ma...@apache.org>
Date: Fri Nov 12 06:29:25 2010 +0000
How to know what input was sent to a mapper? (log the input split where users can see) from Krishna Ramachandran via mahadev)
+++ b/YAHOO-CHANGES.txt
+ How to know what input was sent to a mapper? (log the input
+ split where users can see) from
+
+ (Krishna Ramachandran via mahadev)
+
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java Tue Mar 8 05:52:59 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapred.Merger.S
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
@@ -417,6 +418,7 @@ class MapTask extends Task {
job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString());
job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart());
job.setLong(JobContext.MAP_INPUT_PATH, fileSplit.getLength());
+ job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, fileSplit.toString());
}
}
@@ -621,6 +623,7 @@ class MapTask extends Task {
split = getSplitDetails(new Path(splitIndex.getSplitLocation()),
splitIndex.getStartOffset());
+ job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, split.toString());
org.apache.hadoop.mapreduce.RecordReader<INKEY,INVALUE> input =
new NewTrackingRecordReader<INKEY,INVALUE>
(inputFormat.createRecordReader(split, taskContext), reporter);
@@ -818,6 +821,7 @@ class MapTask extends Task {
LOG.info("soft limit at " + softLimit);
LOG.info("bufstart = " + bufstart + "; bufvoid = " + bufvoid);
LOG.info("kvstart = " + kvstart + "; length = " + maxRec);
+ LOG.info("split details: " + job.get(TTConfig.TT_MAP_INPUT_SPLITINFO));
}
// k/v serialization
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Tue Mar 8 05:52:59 2011
@@ -70,6 +70,8 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.instrumentation";
public static final String TT_MAP_SLOTS =
"mapreduce.tasktracker.map.tasks.maximum";
+ public static final String TT_MAP_INPUT_SPLITINFO =
+ "mapreduce.tasktracker.map.input.splitinfo";
/**
* @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead
*/