You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by jd...@apache.org on 2012/03/15 09:28:38 UTC

svn commit: r1300844 - in /incubator/mrunit/trunk/src: main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java

Author: jdonofrio
Date: Thu Mar 15 08:28:38 2012
New Revision: 1300844

URL: http://svn.apache.org/viewvc?rev=1300844&view=rev
Log:
MRUNIT-67: o.a.h.mrunit.mapreduce.MapReduceDriver should support a combiner

Modified:
    incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
    incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java

Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=1300844&r1=1300843&r2=1300844&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Thu Mar 15 08:28:38 2012
@@ -49,6 +49,7 @@ public class MapReduceDriver<K1, V1, K2 
 
   private Mapper<K1, V1, K2, V2> myMapper;
   private Reducer<K2, V2, K3, V3> myReducer;
+  private Reducer<K2, V2, K2, V2> myCombiner;
   private Counters counters;
 
   public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
@@ -58,6 +59,14 @@ public class MapReduceDriver<K1, V1, K2 
     counters = new Counters();
   }
 
+  public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
+      final Reducer<K2, V2, K3, V3> r, final Reducer<K2, V2, K2, V2> c) {
+    myMapper = m;
+    myReducer = r;
+    myCombiner = c;
+    counters = new Counters();
+  }
+
   public MapReduceDriver() {
     counters = new Counters();
   }
@@ -116,6 +125,36 @@ public class MapReduceDriver<K1, V1, K2 
     return myReducer;
   }
 
+  /**
+   * Sets the reducer object to use as a combiner for this test
+   * 
+   * @param c
+   *          The combiner object to use
+   */
+  public void setCombiner(final Reducer<K2, V2, K2, V2> c) {
+    myCombiner = c;
+  }
+
+  /**
+   * Identical to setCombiner(), but with fluent programming style
+   * 
+   * @param c
+   *          The Combiner to use
+   * @return this
+   */
+  public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCombiner(
+      final Reducer<K2, V2, K2, V2> c) {
+    setCombiner(c);
+    return this;
+  }
+
+  /**
+   * @return the Combiner object being used for this test
+   */
+  public Reducer<K2, V2, K2, V2> getCombiner() {
+    return myCombiner;
+  }
+
   /** @return the counters used in this test */
   public Counters getCounters() {
     return counters;
@@ -221,13 +260,41 @@ public class MapReduceDriver<K1, V1, K2 
     return this;
   }
 
+  /**
+   * The private class to manage starting the reduce phase is used for type
+   * genericity reasons. This class is used in the run() method.
+   */
+  private class ReducePhaseRunner<OUTKEY, OUTVAL> {
+    private List<Pair<OUTKEY, OUTVAL>> runReduce(
+        final List<Pair<K2, List<V2>>> inputs,
+        final Reducer<K2, V2, OUTKEY, OUTVAL> reducer) throws IOException {
+
+      final List<Pair<OUTKEY, OUTVAL>> reduceOutputs = new ArrayList<Pair<OUTKEY, OUTVAL>>();
+
+      for (final Pair<K2, List<V2>> input : inputs) {
+        final K2 inputKey = input.getFirst();
+        final List<V2> inputValues = input.getSecond();
+        final StringBuilder sb = new StringBuilder();
+        formatValueList(inputValues, sb);
+        LOG.debug("Reducing input (" + inputKey.toString() + ", "
+            + sb.toString() + ")");
+
+        reduceOutputs.addAll(ReduceDriver.newReduceDriver(reducer)
+            .withCounters(getCounters()).withConfiguration(configuration)
+            .withInputKey(inputKey).withInputValues(inputValues).run());
+      }
+
+      return reduceOutputs;
+    }
+  }
+
   @Override
   public List<Pair<K3, V3>> run() throws IOException {
     if (inputList.size() == 0) {
       LOG.warn("No inputs configured to send to Mapper and Reducer; this is a trivial test.");
     }
 
-    final List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>();
+    List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>();
 
     // run map component
     for (final Pair<K1, V1> input : inputList) {
@@ -237,23 +304,18 @@ public class MapReduceDriver<K1, V1, K2 
           .withCounters(getCounters()).withConfiguration(configuration).run());
     }
 
-    final List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs);
-    final List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>();
-
-    for (final Pair<K2, List<V2>> input : reduceInputs) {
-      final K2 inputKey = input.getFirst();
-      final List<V2> inputValues = input.getSecond();
-      final StringBuilder sb = new StringBuilder();
-      formatValueList(inputValues, sb);
-      LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString()
-          + ")");
-
-      reduceOutputs.addAll(ReduceDriver.newReduceDriver(myReducer)
-          .withCounters(getCounters()).withConfiguration(configuration)
-          .withInputKey(inputKey).withInputValues(inputValues).run());
+    if (myCombiner != null) {
+      // User has specified a combiner. Run this and replace the mapper outputs
+      // with the result of the combiner.
+      LOG.debug("Starting combine phase with combiner: " + myCombiner);
+      mapOutputs = new ReducePhaseRunner<K2, V2>().runReduce(
+          shuffle(mapOutputs), myCombiner);
     }
 
-    return reduceOutputs;
+    // Run the reduce phase.
+    LOG.debug("Starting reduce phase with reducer: " + myReducer);
+    return new ReducePhaseRunner<K3, V3>().runReduce(shuffle(mapOutputs),
+        myReducer);
   }
 
   @Override
@@ -305,10 +367,6 @@ public class MapReduceDriver<K1, V1, K2 
    * Returns a new MapReduceDriver without having to specify the generic types
    * on the right hand side of the object create statement.
    * 
-   * @param mapper
-   *          passed to MapReduceDriver constructor
-   * @param reducer
-   *          passed to MapReduceDriver constructor
    * @return new MapReduceDriver
    */
   public static <K1, V1, K2 extends Comparable, V2, K3, V3> MapReduceDriver<K1, V1, K2, V2, K3, V3> newMapReduceDriver() {
@@ -329,4 +387,24 @@ public class MapReduceDriver<K1, V1, K2 
       final Mapper<K1, V1, K2, V2> mapper, final Reducer<K2, V2, K3, V3> reducer) {
     return new MapReduceDriver<K1, V1, K2, V2, K3, V3>(mapper, reducer);
   }
+
+  /**
+   * Returns a new MapReduceDriver without having to specify the generic types
+   * on the right hand side of the object create statement.
+   * 
+   * @param mapper
+   *          passed to MapReduceDriver constructor
+   * @param reducer
+   *          passed to MapReduceDriver constructor
+   * @param combiner
+   *          passed to MapReduceDriver constructor
+   * @return new MapReduceDriver
+   */
+  public static <K1, V1, K2 extends Comparable, V2, K3, V3> MapReduceDriver<K1, V1, K2, V2, K3, V3> newMapReduceDriver(
+      final Mapper<K1, V1, K2, V2> mapper,
+      final Reducer<K2, V2, K3, V3> reducer,
+      final Reducer<K2, V2, K2, V2> combiner) {
+    return new MapReduceDriver<K1, V1, K2, V2, K3, V3>(mapper, reducer,
+        combiner);
+  }
 }

Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java?rev=1300844&r1=1300843&r2=1300844&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java Thu Mar 15 08:28:38 2012
@@ -302,6 +302,41 @@ public class TestMapReduceDriver {
     driver.runTest();
   }
 
+  // Test "combining" with an IdentityReducer. Result should be the same.
+  @Test
+  public void testIdentityCombiner() {
+    driver.withCombiner(new Reducer<Text, LongWritable, Text, LongWritable>())
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+        .withInput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+  }
+
+  // Test "combining" with another LongSumReducer. Result should be the same.
+  @Test
+  public void testLongSumCombiner() {
+    driver.withCombiner(new LongSumReducer<Text>())
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+        .withInput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+  }
+
+  // Test "combining" with another LongSumReducer, and with the Reducer
+  // set to IdentityReducer. Result should be the same.
+  @Test
+  public void testLongSumCombinerAndIdentityReduce() {
+    driver.withCombiner(new LongSumReducer<Text>())
+        .withReducer(new Reducer<Text, LongWritable, Text, LongWritable>())
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_A))
+        .withInput(new Text("foo"), new LongWritable(FOO_IN_B))
+        .withInput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("bar"), new LongWritable(BAR_IN))
+        .withOutput(new Text("foo"), new LongWritable(FOO_OUT)).runTest();
+  }
+
   @Test
   public void testNoMapper() {
     driver = MapReduceDriver.newMapReduceDriver();