You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by br...@apache.org on 2012/04/15 22:28:07 UTC
svn commit: r1326419 - in /incubator/mrunit/branches/mrunit-0.9/src:
main/java/org/apache/hadoop/mrunit/mapreduce/
main/java/org/apache/hadoop/mrunit/mapreduce/mock/
test/java/org/apache/hadoop/mrunit/mapreduce/
Author: brock
Date: Sun Apr 15 20:28:07 2012
New Revision: 1326419
URL: http://svn.apache.org/viewvc?rev=1326419&view=rev
Log:
MRUNIT-97: InputSplit causes NullPointerException in mapreduce api
Modified:
incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
incubator/mrunit/branches/mrunit-0.9/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java
Modified: incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java?rev=1326419&r1=1326418&r2=1326419&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java Sun Apr 15 20:28:07 2012
@@ -28,9 +28,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mrunit.MapDriverBase;
import org.apache.hadoop.mrunit.counters.CounterWrapper;
+import org.apache.hadoop.mrunit.mapreduce.mock.MockInputSplit;
import org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper;
import org.apache.hadoop.mrunit.types.Pair;
@@ -209,9 +211,11 @@ public class MapDriver<K1, V1, K2, V2> e
final List<Pair<K1, V1>> inputs = new ArrayList<Pair<K1, V1>>();
inputs.add(new Pair<K1, V1>(inputKey, inputVal));
+ final InputSplit inputSplit = new MockInputSplit();
+
try {
final MockMapContextWrapper<K1, V1, K2, V2> wrapper = new MockMapContextWrapper<K1, V1, K2, V2>(
- inputs, getCounters(), getConfiguration());
+ inputs, getCounters(), getConfiguration(), inputSplit);
final Mapper<K1, V1, K2, V2>.Context context = wrapper.getMockContext();
myMapper.run(context);
Modified: incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java?rev=1326419&r1=1326418&r2=1326419&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java Sun Apr 15 20:28:07 2012
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mrunit.types.Pair;
import org.mockito.invocation.InvocationOnMock;
@@ -50,12 +51,14 @@ public class MockMapContextWrapper<KEYIN
protected final List<Pair<KEYIN, VALUEIN>> inputs;
protected Pair<KEYIN, VALUEIN> currentKeyValue;
protected final Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context;
-
+ protected InputSplit inputSplit;
+
public MockMapContextWrapper(final List<Pair<KEYIN, VALUEIN>> inputs,
- final Counters counters, final Configuration conf) throws IOException,
- InterruptedException {
+ final Counters counters, final Configuration conf, final InputSplit inputSplit)
+ throws IOException, InterruptedException {
super(counters, conf);
this.inputs = inputs;
+ this.inputSplit = inputSplit;
context = create();
}
@@ -94,6 +97,12 @@ public class MockMapContextWrapper<KEYIN
return currentKeyValue.getSecond();
}
});
+ when(context.getInputSplit()).thenAnswer(new Answer<InputSplit>() {
+ @Override
+ public InputSplit answer(InvocationOnMock invocation) throws Throwable {
+ return inputSplit;
+ }
+ });
return context;
}
Modified: incubator/mrunit/branches/mrunit-0.9/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java?rev=1326419&r1=1326418&r2=1326419&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapDriver.java Sun Apr 15 20:28:07 2012
@@ -26,9 +26,11 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
import org.apache.hadoop.mrunit.ExpectedSuppliedException;
import org.apache.hadoop.mrunit.types.Pair;
@@ -244,6 +246,24 @@ public class TestMapDriver {
}
@Test
+ public void testInputSplitDetails() {
+ final MapDriver<NullWritable, NullWritable, Text, LongWritable> driver =
+ MapDriver.newMapDriver(new InputSplitDetailMapper());
+ driver.withInput(NullWritable.get(), NullWritable.get())
+ .withOutput(new Text("somefile"), new LongWritable(0L)).runTest();
+ }
+
+ public static class InputSplitDetailMapper
+ extends Mapper<NullWritable, NullWritable, Text, LongWritable> {
+ protected void map(NullWritable key, NullWritable value, Context context)
+ throws IOException, InterruptedException {
+ FileSplit split = (FileSplit)context.getInputSplit();
+ context.write(new Text(split.getPath().toString()),
+ new LongWritable(split.getLength()));
+ }
+ }
+
+ @Test
public void testNoMapper() {
driver = MapDriver.newMapDriver();
thrown.expectMessage(IllegalStateException.class,