You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/04/15 00:00:33 UTC
svn commit: r648022 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/MapTask.java
Author: cdouglas
Date: Mon Apr 14 15:00:32 2008
New Revision: 648022
URL: http://svn.apache.org/viewvc?rev=648022&view=rev
Log:
HADOOP-3229. Report progress when collecting records from the mapper and
the combiner. Contributed by Doug Cutting.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=648022&r1=648021&r2=648022&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Apr 14 15:00:32 2008
@@ -579,6 +579,9 @@
HADOOP-3204. Fixes a problem to do with ReduceTask's LocalFSMerger not
catching Throwable. (Amar Ramesh Kamat via ddas)
+ HADOOP-3229. Report progress when collecting records from the mapper and
+ the combiner. (Doug Cutting via cdouglas)
+
Release 0.16.3 - 2008-04-16
BUG FIXES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?rev=648022&r1=648021&r2=648022&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Mon Apr 14 15:00:32 2008
@@ -263,6 +263,7 @@
}
public void collect(K key, V value) throws IOException {
+ reporter.progress();
this.out.write(key, value);
}
@@ -406,13 +407,14 @@
deflateFilter = null;
}
combineCollector = (null != combinerClass)
- ? new CombineOutputCollector()
+ ? new CombineOutputCollector(reporter)
: null;
}
@SuppressWarnings("unchecked")
public synchronized void collect(Object key, Object value)
throws IOException {
+ reporter.progress();
if (key.getClass() != keyClass) {
throw new IOException("Type mismatch in key from map: expected "
+ keyClass.getName() + ", recieved "
@@ -1063,12 +1065,17 @@
* OutputCollector for the combiner.
*/
private static class CombineOutputCollector implements OutputCollector {
+ private Reporter reporter;
private SequenceFile.Writer writer;
+ public CombineOutputCollector(Reporter reporter) {
+ this.reporter = reporter;
+ }
public synchronized void setWriter(SequenceFile.Writer writer) {
this.writer = writer;
}
public synchronized void collect(Object key, Object value)
throws IOException {
+ reporter.progress();
writer.append(key, value);
}
}