You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/12/09 11:09:17 UTC
svn commit: r888756 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/
src/test/mapred/org/apache/hadoop/mapreduce/
Author: cdouglas
Date: Wed Dec 9 10:09:16 2009
New Revision: 888756
URL: http://svn.apache.org/viewvc?rev=888756&view=rev
Log:
MAPREDUCE-952. Remove inadvertently reintroduced Task.Counter enum. Contributed by Jothi Padmanabhan
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/ReduceTask.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/Task.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Wed Dec 9 10:09:16 2009
@@ -863,3 +863,6 @@
Padmanabhan via cdouglas)
MAPREDUCE-1267. Fix typo in mapred-default.xml. (Todd Lipcon via cdouglas)
+
+ MAPREDUCE-952. Remove inadvertently reintroduced Task.Counter enum. (Jothi
+ Padmanabhan via cdouglas)
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/ReduceTask.java?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/ReduceTask.java Wed Dec 9 10:09:16 2009
@@ -45,6 +45,7 @@
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.task.reduce.Shuffle;
import org.apache.hadoop.util.Progress;
@@ -77,19 +78,19 @@
private Progress sortPhase;
private Progress reducePhase;
private Counters.Counter shuffledMapsCounter =
- getCounters().findCounter(Counter.SHUFFLED_MAPS);
+ getCounters().findCounter(TaskCounter.SHUFFLED_MAPS);
private Counters.Counter reduceShuffleBytes =
- getCounters().findCounter(Counter.REDUCE_SHUFFLE_BYTES);
+ getCounters().findCounter(TaskCounter.REDUCE_SHUFFLE_BYTES);
private Counters.Counter reduceInputKeyCounter =
- getCounters().findCounter(Counter.REDUCE_INPUT_GROUPS);
+ getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
private Counters.Counter reduceInputValueCounter =
- getCounters().findCounter(Counter.REDUCE_INPUT_RECORDS);
+ getCounters().findCounter(TaskCounter.REDUCE_INPUT_RECORDS);
private Counters.Counter reduceOutputCounter =
- getCounters().findCounter(Counter.REDUCE_OUTPUT_RECORDS);
+ getCounters().findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS);
private Counters.Counter reduceCombineInputCounter =
- getCounters().findCounter(Counter.COMBINE_INPUT_RECORDS);
+ getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
private Counters.Counter reduceCombineOutputCounter =
- getCounters().findCounter(Counter.COMBINE_OUTPUT_RECORDS);
+ getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
// A custom comparator for map output files. Here the ordering is determined
// by the file's size and path. In case of files with same size and different
@@ -237,9 +238,9 @@
super(in, comparator, keyClass, valClass, conf, reporter);
this.umbilical = umbilical;
this.skipGroupCounter =
- reporter.getCounter(Counter.REDUCE_SKIPPED_GROUPS);
+ reporter.getCounter(TaskCounter.REDUCE_SKIPPED_GROUPS);
this.skipRecCounter =
- reporter.getCounter(Counter.REDUCE_SKIPPED_RECORDS);
+ reporter.getCounter(TaskCounter.REDUCE_SKIPPED_RECORDS);
this.toWriteSkipRecs = toWriteSkipRecs() &&
SkipBadRecords.getSkipOutputPath(conf)!=null;
this.keyClass = keyClass;
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/Task.java?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/Task.java Wed Dec 9 10:09:16 2009
@@ -48,8 +48,8 @@
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
-import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -65,27 +65,6 @@
private static final Log LOG =
LogFactory.getLog(Task.class);
- // Counters used by Task subclasses
- protected static enum Counter {
- MAP_INPUT_RECORDS,
- MAP_OUTPUT_RECORDS,
- MAP_SKIPPED_RECORDS,
- MAP_INPUT_BYTES,
- MAP_OUTPUT_BYTES,
- COMBINE_INPUT_RECORDS,
- COMBINE_OUTPUT_RECORDS,
- REDUCE_INPUT_GROUPS,
- REDUCE_SHUFFLE_BYTES,
- REDUCE_INPUT_RECORDS,
- REDUCE_OUTPUT_RECORDS,
- REDUCE_SKIPPED_GROUPS,
- REDUCE_SKIPPED_RECORDS,
- SPILLED_RECORDS,
- FAILED_SHUFFLE,
- SHUFFLED_MAPS,
- MERGED_MAP_OUTPUTS,
- }
-
public static String MERGED_OUTPUT_PREFIX = ".merged";
@@ -164,9 +143,12 @@
public Task() {
taskStatus = TaskStatus.createTaskStatus(isMapTask());
taskId = new TaskAttemptID();
- spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS);
- failedShuffleCounter = counters.findCounter(Counter.FAILED_SHUFFLE);
- mergedMapOutputsCounter = counters.findCounter(Counter.MERGED_MAP_OUTPUTS);
+ spilledRecordsCounter =
+ counters.findCounter(TaskCounter.SPILLED_RECORDS);
+ failedShuffleCounter =
+ counters.findCounter(TaskCounter.FAILED_SHUFFLE);
+ mergedMapOutputsCounter =
+ counters.findCounter(TaskCounter.MERGED_MAP_OUTPUTS);
}
public Task(String jobFile, TaskAttemptID taskId, int partition,
@@ -185,8 +167,9 @@
TaskStatus.Phase.SHUFFLE,
counters);
spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS);
- failedShuffleCounter = counters.findCounter(Counter.FAILED_SHUFFLE);
- mergedMapOutputsCounter = counters.findCounter(Counter.MERGED_MAP_OUTPUTS);
+ failedShuffleCounter = counters.findCounter(TaskCounter.FAILED_SHUFFLE);
+ mergedMapOutputsCounter =
+ counters.findCounter(TaskCounter.MERGED_MAP_OUTPUTS);
}
////////////////////////////////////////////
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.java?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.java Wed Dec 9 10:09:16 2009
@@ -32,5 +32,8 @@
REDUCE_OUTPUT_RECORDS,
REDUCE_SKIPPED_GROUPS,
REDUCE_SKIPPED_RECORDS,
- SPILLED_RECORDS
+ SPILLED_RECORDS,
+ SHUFFLED_MAPS,
+ FAILED_SHUFFLE,
+ MERGED_MAP_OUTPUTS
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/TaskCounter.properties Wed Dec 9 10:09:16 2009
@@ -15,4 +15,6 @@
REDUCE_SKIPPED_RECORDS.name= Reduce skipped records
REDUCE_SKIPPED_GROUPS.name= Reduce skipped groups
SPILLED_RECORDS.name= Spilled Records
-
+SHUFFLED_MAPS.name= Shuffled Maps
+FAILED_SHUFFLE.name= Failed Shuffles
+MERGED_MAP_OUTPUTS.name= Merged Map outputs
Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java?rev=888756&r1=888755&r2=888756&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java Wed Dec 9 10:09:16 2009
@@ -162,10 +162,16 @@
"REDUCE_OUTPUT_RECORDS").getValue();
long reduceGrps = ctrs.findCounter(COUNTER_GROUP,
"REDUCE_INPUT_GROUPS").getValue();
+ long mergedMapOutputs = ctrs.findCounter(COUNTER_GROUP,
+ "MERGED_MAP_OUTPUTS").getValue();
+ long shuffledMaps = ctrs.findCounter(COUNTER_GROUP,
+ "SHUFFLED_MAPS").getValue();
assertEquals("map out = combine in", mapOut, combineIn);
assertEquals("combine out = reduce in", combineOut, reduceIn);
assertTrue("combine in > combine out", combineIn > combineOut);
assertEquals("reduce groups = reduce out", reduceGrps, reduceOut);
+ assertEquals("Mismatch in mergedMapOutputs", mergedMapOutputs, 2);
+ assertEquals("Mismatch in shuffledMaps", shuffledMaps, 2);
String group = "Random Group";
CounterGroup ctrGrp = ctrs.getGroup(group);
assertEquals(0, ctrGrp.size());