You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by jd...@apache.org on 2012/03/15 09:28:38 UTC
svn commit: r1300844 - in /incubator/mrunit/trunk/src:
main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
Author: jdonofrio
Date: Thu Mar 15 08:28:38 2012
New Revision: 1300844
URL: http://svn.apache.org/viewvc?rev=1300844&view=rev
Log:
MRUNIT-67: o.a.h.mrunit.mapreduce.MapReduceDriver should support a combiner
Modified:
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=1300844&r1=1300843&r2=1300844&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Thu Mar 15 08:28:38 2012
@@ -49,6 +49,7 @@ public class MapReduceDriver<K1, V1, K2
private Mapper<K1, V1, K2, V2> myMapper;
private Reducer<K2, V2, K3, V3> myReducer;
+ private Reducer<K2, V2, K2, V2> myCombiner;
private Counters counters;
public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
@@ -58,6 +59,14 @@ public class MapReduceDriver<K1, V1, K2
counters = new Counters();
}
+ public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
+ final Reducer<K2, V2, K3, V3> r, final Reducer<K2, V2, K2, V2> c) {
+ myMapper = m;
+ myReducer = r;
+ myCombiner = c;
+ counters = new Counters();
+ }
+
public MapReduceDriver() {
counters = new Counters();
}
@@ -116,6 +125,36 @@ public class MapReduceDriver<K1, V1, K2
return myReducer;
}
+ /**
+ * Sets the reducer object to use as a combiner for this test
+ *
+ * @param c
+ * The combiner object to use
+ */
+ public void setCombiner(final Reducer<K2, V2, K2, V2> c) {
+ myCombiner = c;
+ }
+
+ /**
+ * Identical to setCombiner(), but with fluent programming style
+ *
+ * @param c
+ * The Combiner to use
+ * @return this
+ */
+ public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCombiner(
+ final Reducer<K2, V2, K2, V2> c) {
+ setCombiner(c);
+ return this;
+ }
+
+ /**
+ * @return the Combiner object being used for this test
+ */
+ public Reducer<K2, V2, K2, V2> getCombiner() {
+ return myCombiner;
+ }
+
/** @return the counters used in this test */
public Counters getCounters() {
return counters;
@@ -221,13 +260,41 @@ public class MapReduceDriver<K1, V1, K2
return this;
}
+ /**
+ * The private class to manage starting the reduce phase is used for type
+ * genericity reasons. This class is used in the run() method.
+ */
+ private class ReducePhaseRunner<OUTKEY, OUTVAL> {
+ private List<Pair<OUTKEY, OUTVAL>> runReduce(
+ final List<Pair<K2, List<V2>>> inputs,
+ final Reducer<K2, V2, OUTKEY, OUTVAL> reducer) throws IOException {
+
+ final List<Pair<OUTKEY, OUTVAL>> reduceOutputs = new ArrayList<Pair<OUTKEY, OUTVAL>>();
+
+ for (final Pair<K2, List<V2>> input : inputs) {
+ final K2 inputKey = input.getFirst();
+ final List<V2> inputValues = input.getSecond();
+ final StringBuilder sb = new StringBuilder();
+ formatValueList(inputValues, sb);
+ LOG.debug("Reducing input (" + inputKey.toString() + ", "
+ + sb.toString() + ")");
+
+ reduceOutputs.addAll(ReduceDriver.newReduceDriver(reducer)
+ .withCounters(getCounters()).withConfiguration(configuration)
+ .withInputKey(inputKey).withInputValues(inputValues).run());
+ }
+
+ return reduceOutputs;
+ }
+ }
+
@Override
public List<Pair<K3, V3>> run() throws IOException {
if (inputList.size() == 0) {
LOG.warn("No inputs configured to send to Mapper and Reducer; this is a trivial test.");
}
- final List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>();
+ List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>();
// run map component
for (final Pair<K1, V1> input : inputList) {
@@ -237,23 +304,18 @@ public class MapReduceDriver<K1, V1, K2
.withCounters(getCounters()).withConfiguration(configuration).run());
}
- final List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs);
- final List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>();
-
- for (final Pair<K2, List<V2>> input : reduceInputs) {
- final K2 inputKey = input.getFirst();
- final List<V2> inputValues = input.getSecond();
- final StringBuilder sb = new StringBuilder();
- formatValueList(inputValues, sb);
- LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString()
- + ")");
-
- reduceOutputs.addAll(ReduceDriver.newReduceDriver(myReducer)
- .withCounters(getCounters()).withConfiguration(configuration)
- .withInputKey(inputKey).withInputValues(inputValues).run());
+ if (myCombiner != null) {
+ // User has specified a combiner. Run this and replace the mapper outputs
+ // with the result of the combiner.
+ LOG.debug("Starting combine phase with combiner: " + myCombiner);
+ mapOutputs = new ReducePhaseRunner<K2, V2>().runReduce(
+ shuffle(mapOutputs), myCombiner);
}
- return reduceOutputs;
+ // Run the reduce phase.
+ LOG.debug("Starting reduce phase with reducer: " + myReducer);
+ return new ReducePhaseRunner<K3, V3>().runReduce(shuffle(mapOutputs),
+ myReducer);
}
@Override
@@ -305,10 +367,6 @@ public class MapReduceDriver<K1, V1, K2
* Returns a new MapReduceDriver without having to specify the generic types
* on the right hand side of the object create statement.
*
- * @param mapper
- * passed to MapReduceDriver constructor
- * @param reducer
- * passed to MapReduceDriver constructor
* @return new MapReduceDriver
*/
public static <K1, V1, K2 extends Comparable, V2, K3, V3> MapReduceDriver<K1, V1, K2, V2, K3, V3> newMapReduceDriver() {
@@ -329,4 +387,24 @@ public class MapReduceDriver<K1, V1, K2
final Mapper<K1, V1, K2, V2> mapper, final Reducer<K2, V2, K3, V3> reducer) {
return new MapReduceDriver<K1, V1, K2, V2, K3, V3>(mapper, reducer);
}
+
+ /**
+ * Returns a new MapReduceDriver without having to specify the generic types
+ * on the right hand side of the object create statement.
+ *
+ * @param mapper
+ * passed to MapReduceDriver constructor
+ * @param reducer
+ * passed to MapReduceDriver constructor
+ * @param combiner
+ * passed to MapReduceDriver constructor
+ * @return new MapReduceDriver
+ */
+ public static <K1, V1, K2 extends Comparable, V2, K3, V3> MapReduceDriver<K1, V1, K2, V2, K3, V3> newMapReduceDriver(
+ final Mapper<K1, V1, K2, V2> mapper,
+ final Reducer<K2, V2, K3, V3> reducer,
+ final Reducer<K2, V2, K2, V2> combiner) {
+ return new MapReduceDriver<K1, V1, K2, V2, K3, V3>(mapper, reducer,
+ combiner);
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java?rev=1300844&r1=1300843&r2=1300844&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java Thu Mar 15 08:28:38 2012
@@ -302,6 +302,41 @@ public class TestMapReduceDriver {
driver.runTest();
}
+ // Test "combining" with an IdentityReducer. Result should be the same.
+ @Test
+ public void testIdentityCombiner() {
+ driver.withCombiner(new Reducer<Text, LongWritable, Text, LongWritable>())
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+ .withInput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+ }
+
+ // Test "combining" with another LongSumReducer. Result should be the same.
+ @Test
+ public void testLongSumCombiner() {
+ driver.withCombiner(new LongSumReducer<Text>())
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+ .withInput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+ }
+
+ // Test "combining" with another LongSumReducer, and with the Reducer
+ // set to IdentityReducer. Result should be the same.
+ @Test
+ public void testLongSumCombinerAndIdentityReduce() {
+ driver.withCombiner(new LongSumReducer<Text>())
+ .withReducer(new Reducer<Text, LongWritable, Text, LongWritable>())
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+ .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+ .withInput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+ .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+ }
+
@Test
public void testNoMapper() {
driver = MapReduceDriver.newMapReduceDriver();