You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by jd...@apache.org on 2012/03/15 08:51:03 UTC
svn commit: r1300829 - in /incubator/mrunit/trunk/src:
main/java/org/apache/hadoop/mrunit/ test/java/org/apache/hadoop/mrunit/
test/java/org/apache/hadoop/mrunit/mapreduce/
Author: jdonofrio
Date: Thu Mar 15 07:51:02 2012
New Revision: 1300829
URL: http://svn.apache.org/viewvc?rev=1300829&view=rev
Log:
MRUNIT-86: Configuration not passed to individual mappers/reducers in mapred MapReducerDriver, PipelineMapReduceDriver
Modified:
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestPipelineMapReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java Thu Mar 15 07:51:02 2012
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
@@ -219,6 +220,18 @@ public class MapDriver<K1, V1, K2, V2> e
}
/**
+ * @param configuration
+ * The configuration object that will given to the mapper associated
+ * with the driver
+ * @return this object for fluent coding
+ */
+ public MapDriver<K1, V1, K2, V2> withConfiguration(
+ final Configuration configuration) {
+ setConfiguration(configuration);
+ return this;
+ }
+
+ /**
* Returns a new MapDriver without having to specify the generic types on the
* right hand side of the object create statement.
*
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Mapper;
@@ -281,8 +282,8 @@ public class MapReduceDriver<K1, V1, K2
+ sb.toString() + ")");
reduceOutputs.addAll(ReduceDriver.newReduceDriver(reducer)
- .withCounters(getCounters()).withInputKey(inputKey)
- .withInputValues(inputValues).run());
+ .withCounters(getCounters()).withConfiguration(configuration)
+ .withInputKey(inputKey).withInputValues(inputValues).run());
}
return reduceOutputs;
@@ -302,7 +303,7 @@ public class MapReduceDriver<K1, V1, K2
LOG.debug("Mapping input " + input.toString() + ")");
mapOutputs.addAll(MapDriver.newMapDriver(myMapper).withInput(input)
- .withCounters(getCounters()).run());
+ .withCounters(getCounters()).withConfiguration(configuration).run());
}
if (myCombiner != null) {
@@ -325,6 +326,18 @@ public class MapReduceDriver<K1, V1, K2
}
/**
+ * @param configuration
+ * The configuration object that will given to the mapper and reducer
+ * associated with the driver
+ * @return this driver object for fluent coding
+ */
+ public MapReduceDriver<K1, V1, K2, V2, K3, V3> withConfiguration(
+ final Configuration configuration) {
+ setConfiguration(configuration);
+ return this;
+ }
+
+ /**
* Identical to {@link #setKeyGroupingComparator(RawComparator)}, but with a
* fluent programming style
*
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Mapper;
@@ -369,6 +370,7 @@ public class PipelineMapReduceDriver<K1,
job.getFirst(), job.getSecond());
mrDriver.setCounters(getCounters());
+ mrDriver.setConfiguration(configuration);
// Add the inputs from the user, or from the previous stage of the
// pipeline.
@@ -406,6 +408,18 @@ public class PipelineMapReduceDriver<K1,
}
/**
+ * @param configuration
+ * The configuration object that will given to the mappers and
+ * reducers associated with the driver
+ * @return this driver object for fluent coding
+ */
+ public PipelineMapReduceDriver<K1, V1, K2, V2> withConfiguration(
+ final Configuration configuration) {
+ setConfiguration(configuration);
+ return this;
+ }
+
+ /**
* Returns a new PipelineMapReduceDriver without having to specify the generic
* types on the right hand side of the object create statement.
*
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reducer;
@@ -229,6 +230,18 @@ public class ReduceDriver<K1, V1, K2, V2
}
/**
+ * @param configuration
+ * The configuration object that will given to the reducer associated
+ * with the driver
+ * @return this object for fluent coding
+ */
+ public ReduceDriver<K1, V1, K2, V2> withConfiguration(
+ final Configuration configuration) {
+ setConfiguration(configuration);
+ return this;
+ }
+
+ /**
* Returns a new ReduceDriver without having to specify the generic types on
* the right hand side of the object create statement.
*
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java Thu Mar 15 07:51:02 2012
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -238,4 +240,17 @@ public class TestMapDriver {
+ "expected: class org.apache.hadoop.io.Text actual: class org.apache.hadoop.io.LongWritable)");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final MapDriver<Integer, IntWritable, Integer, IntWritable> driver = MapDriver
+ .newMapDriver(new IdentityMapper<Integer, IntWritable>())
+ .withConfiguration(conf);
+ driver.setInput(1, new IntWritable(2));
+ driver.addOutput(1, new IntWritable(2));
+ driver.runTest();
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -25,6 +25,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
@@ -341,4 +343,18 @@ public class TestMapReduceDriver {
"No Reducer class was provided");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final MapReduceDriver<Integer, IntWritable, Integer, IntWritable, Integer, IntWritable> driver = MapReduceDriver
+ .newMapReduceDriver(new IdentityMapper<Integer, IntWritable>(),
+ new IdentityReducer<Integer, IntWritable>())
+ .withConfiguration(conf);
+ driver.addInput(1, new IntWritable(2));
+ driver.addOutput(1, new IntWritable(2));
+ driver.runTest();
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestPipelineMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestPipelineMapReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestPipelineMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestPipelineMapReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.lib.IdentityMapper;
@@ -160,4 +162,22 @@ public class TestPipelineMapReduceDriver
"No Reducer class was provided for stage 2 of 3");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final PipelineMapReduceDriver<Integer, IntWritable, Integer, IntWritable> driver = PipelineMapReduceDriver
+ .newPipelineMapReduceDriver();
+ driver.addMapReduce(new IdentityMapper<Integer, IntWritable>(),
+ new IdentityReducer<Integer, IntWritable>());
+ driver.addMapReduce(new IdentityMapper<Integer, IntWritable>(),
+ new IdentityReducer<Integer, IntWritable>());
+ driver.withConfiguration(conf);
+
+ driver.addInput(1, new IntWritable(2));
+ driver.addOutput(1, new IntWritable(2));
+ driver.runTest();
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -223,16 +223,14 @@ public class TestReduceDriver {
}
@Test
- public void testJavaSerialization() {
- final ReduceDriver<Integer, Integer, Integer, Integer> driver = ReduceDriver
- .newReduceDriver(new IdentityReducer<Integer, Integer>());
+ public void testConf() {
final Configuration conf = new Configuration();
- conf.set("io.serializations",
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
"org.apache.hadoop.io.serializer.JavaSerialization");
- driver.setConfiguration(conf);
- driver.withInputKey(1).withInputValue(2).withInputValue(3)
- .withInputValue(4).withOutput(1, 2).withOutput(1, 3).withOutput(1, 4)
- .runTest();
+ final ReduceDriver<Integer, Integer, Integer, Integer> driver = ReduceDriver
+ .newReduceDriver(new IdentityReducer<Integer, Integer>())
+ .withConfiguration(conf);
+ driver.withInputKey(1).withInputValue(2).withOutput(1, 2).runTest();
}
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java Thu Mar 15 07:51:02 2012
@@ -25,9 +25,11 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
import org.apache.hadoop.mrunit.ExpectedSuppliedException;
import org.apache.hadoop.mrunit.types.Pair;
import org.junit.Before;
@@ -189,4 +191,17 @@ public class TestMapDriver {
"No Mapper class was provided");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final MapDriver<Integer, IntWritable, IntWritable, Integer> driver = MapDriver
+ .newMapDriver(new InverseMapper<Integer, IntWritable>())
+ .withConfiguration(conf);
+ driver.setInput(1, new IntWritable(2));
+ driver.addOutput(new IntWritable(2), 1);
+ driver.runTest();
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -25,12 +25,15 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
+import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
import org.apache.hadoop.mrunit.ExpectedSuppliedException;
import org.apache.hadoop.mrunit.mapreduce.TestMapDriver.ConfigurationMapper;
@@ -316,4 +319,16 @@ public class TestMapReduceDriver {
"No Reducer class was provided");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final MapReduceDriver<IntWritable, Integer, Integer, IntWritable, Integer, IntWritable> driver = MapReduceDriver
+ .newMapReduceDriver(new InverseMapper<IntWritable, Integer>(),
+ new IntSumReducer<Integer>()).withConfiguration(conf);
+ driver.withInput(new IntWritable(1), 2).withOutput(2, new IntWritable(1))
+ .runTest();
+ }
}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java?rev=1300829&r1=1300828&r2=1300829&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestReduceDriver.java Thu Mar 15 07:51:02 2012
@@ -27,10 +27,12 @@ import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
import org.apache.hadoop.mrunit.ExpectedSuppliedException;
import org.apache.hadoop.mrunit.types.Pair;
@@ -246,4 +248,15 @@ public class TestReduceDriver {
"No Reducer class was provided");
driver.runTest();
}
+
+ @Test
+ public void testConf() {
+ final Configuration conf = new Configuration();
+ conf.setStrings("io.serializations", conf.get("io.serializations"),
+ "org.apache.hadoop.io.serializer.JavaSerialization");
+ final ReduceDriver<Integer, IntWritable, Integer, IntWritable> driver = ReduceDriver
+ .newReduceDriver(new IntSumReducer<Integer>()).withConfiguration(conf);
+ driver.withInputKey(1).withInputValue(new IntWritable(2))
+ .withOutput(1, new IntWritable(2)).runTest();
+ }
}