You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/11/05 07:05:32 UTC
svn commit: r833006 - in /hadoop/mapreduce/trunk: ./
src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/
src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/
src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/
Author: cdouglas
Date: Thu Nov 5 06:05:32 2009
New Revision: 833006
URL: http://svn.apache.org/viewvc?rev=833006&view=rev
Log:
MAPREDUCE-1128. Fix MRUnit to prohibit iterating over values twice. Contributed by Aaron Kimball
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContext.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=833006&r1=833005&r2=833006&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Nov 5 06:05:32 2009
@@ -66,6 +66,9 @@
MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned.
(sharad)
+ MAPREDUCE-1128. Fix MRUnit to prohibit iterating over values twice. (Aaron
+ Kimball via cdouglas)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContext.java?rev=833006&r1=833005&r2=833006&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContext.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContext.java Thu Nov 5 06:05:32 2009
@@ -63,20 +63,41 @@
private class InspectableIterable implements Iterable<VALUEIN> {
private Iterable<VALUEIN> base;
private VALUEIN lastVal;
+ private boolean used; // if true, don't re-iterate.
public InspectableIterable(final Iterable<VALUEIN> baseCollection) {
this.base = baseCollection;
}
public Iterator<VALUEIN> iterator() {
- return new InspectableIterator(this.base.iterator());
+ if (used) {
+ return new NullIterator();
+ } else {
+ used = true;
+ return new InspectableIterator(this.base.iterator());
+ }
}
public VALUEIN getLastVal() {
return lastVal;
}
- private class InspectableIterator
+ private class NullIterator
+ extends ReduceContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.ValueIterator
+ implements Iterator<VALUEIN> {
+ public VALUEIN next() {
+ return null;
+ }
+
+ public boolean hasNext() {
+ return false;
+ }
+
+ public void remove() {
+ }
+ }
+
+ private class InspectableIterator
extends ReduceContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.ValueIterator
implements Iterator<VALUEIN> {
private Iterator<VALUEIN> iter;
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestReduceDriver.java?rev=833006&r1=833005&r2=833006&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestReduceDriver.java Thu Nov 5 06:05:32 2009
@@ -22,13 +22,17 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import junit.framework.TestCase;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.LongSumReducer;
import org.apache.hadoop.mrunit.types.Pair;
import org.junit.Before;
@@ -222,5 +226,45 @@
// expected.
}
}
+
+ /**
+ * Reducer that counts its values twice; the second iteration
+ * according to mapreduce semantics should be empty.
+ */
+ private static class DoubleIterReducer<K, V>
+ extends MapReduceBase implements Reducer<K, V, K, LongWritable> {
+ public void reduce(K key, Iterator<V> values,
+ OutputCollector<K, LongWritable> out, Reporter r) throws IOException {
+ long count = 0;
+
+ while (values.hasNext()) {
+ count++;
+ values.next();
+ }
+
+ // This time around, iteration should yield no values.
+ while (values.hasNext()) {
+ count++;
+ values.next();
+ }
+ out.collect(key, new LongWritable(count));
+ }
+ }
+
+ @Test
+ public void testDoubleIteration() {
+ reducer = new DoubleIterReducer<Text, LongWritable>();
+ driver = new ReduceDriver<Text, LongWritable, Text, LongWritable>(
+ reducer);
+
+ driver
+ .withInputKey(new Text("foo"))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withOutput(new Text("foo"), new LongWritable(4))
+ .runTest();
+ }
}
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java?rev=833006&r1=833005&r2=833006&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java Thu Nov 5 06:05:32 2009
@@ -223,5 +223,43 @@
// expected.
}
}
+
+ /**
+ * Reducer that counts its values twice; the second iteration
+ * according to mapreduce semantics should be empty.
+ */
+ private static class DoubleIterReducer<K, V>
+ extends Reducer<K, V, K, LongWritable> {
+ public void reduce(K key, Iterable<V> values, Context c)
+ throws IOException, InterruptedException {
+ long count = 0;
+
+ for (V val : values) {
+ count++;
+ }
+
+ // This time around, iteration should yield no values.
+ for (V val : values) {
+ count++;
+ }
+ c.write(key, new LongWritable(count));
+ }
+ }
+
+ @Test
+ public void testDoubleIteration() {
+ reducer = new DoubleIterReducer<Text, LongWritable>();
+ driver = new ReduceDriver<Text, LongWritable, Text, LongWritable>(
+ reducer);
+
+ driver
+ .withInputKey(new Text("foo"))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withInputValue(new LongWritable(1))
+ .withOutput(new Text("foo"), new LongWritable(4))
+ .runTest();
+ }
}