You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by br...@apache.org on 2012/06/16 18:47:01 UTC
svn commit: r1350958 - in /mrunit/trunk/src:
main/java/org/apache/hadoop/mrunit/
main/java/org/apache/hadoop/mrunit/mapreduce/
test/java/org/apache/hadoop/mrunit/
test/java/org/apache/hadoop/mrunit/mapreduce/
Author: brock
Date: Sat Jun 16 16:47:01 2012
New Revision: 1350958
URL: http://svn.apache.org/viewvc?rev=1350958&view=rev
Log:
MRUNIT-118: InputSplit should support a configurable filename
Modified:
mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java
mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
Modified: mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=1350958&r1=1350957&r2=1350958&view=diff
==============================================================================
--- mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java (original)
+++ mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java Sat Jun 16 16:47:01 2012
@@ -343,7 +343,7 @@ public class MapReduceDriver<K1, V1, K2,
mapOutputs.addAll(MapDriver.newMapDriver(myMapper)
.withConfiguration(configuration).withCounters(getCounters())
- .withInput(input).run());
+ .withInput(input).withMapInputPath(getMapInputPath()).run());
}
if (myCombiner != null) {
Modified: mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=1350958&r1=1350957&r2=1350958&view=diff
==============================================================================
--- mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (original)
+++ mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Sat Jun 16 16:47:01 2012
@@ -342,7 +342,7 @@ public class MapReduceDriver<K1, V1, K2,
mapOutputs.addAll(MapDriver.newMapDriver(myMapper)
.withCounters(getCounters()).withConfiguration(configuration)
- .withInput(input).run());
+ .withInput(input).withMapInputPath(getMapInputPath()).run());
}
if (myCombiner != null) {
Modified: mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java?rev=1350958&r1=1350957&r2=1350958&view=diff
==============================================================================
--- mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java (original)
+++ mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapReduceDriver.java Sat Jun 16 16:47:01 2012
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.mrunit;
-import static org.apache.hadoop.mrunit.ExtendedAssert.assertListEquals;
-import static org.junit.Assert.assertEquals;
+import static org.apache.hadoop.mrunit.ExtendedAssert.*;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,11 +26,14 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
@@ -477,4 +480,35 @@ public class TestMapReduceDriver {
driver.withOutput(new LongWritable(), new Text("a\t3"));
driver.runTest();
}
+
+ private static class InputPathStoringMapper extends MapReduceBase implements
+ Mapper<Text, LongWritable, Text, LongWritable> {
+ private Path mapInputPath;
+
+ @Override
+ public void map(Text key, LongWritable value,
+ OutputCollector<Text, LongWritable> output, Reporter reporter)
+ throws IOException {
+ if (reporter.getInputSplit() instanceof FileSplit) {
+ mapInputPath = ((FileSplit) reporter.getInputSplit()).getPath();
+ }
+ }
+
+ private Path getMapInputPath() {
+ return mapInputPath;
+ }
+ }
+
+ @Test
+ public void testMapInputFile() {
+ InputPathStoringMapper mapper = new InputPathStoringMapper();
+ Path mapInputPath = new Path("myfile");
+ driver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
+ driver.setMapInputPath(mapInputPath);
+ assertEquals(mapInputPath.getName(), driver.getMapInputPath().getName());
+ driver.withInput(new Text("a"), new LongWritable(1));
+ driver.runTest();
+ assertNotNull(mapper.getMapInputPath());
+ assertEquals(mapInputPath.getName(), mapper.getMapInputPath().getName());
+ }
}
Modified: mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java
URL: http://svn.apache.org/viewvc/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java?rev=1350958&r1=1350957&r2=1350958&view=diff
==============================================================================
--- mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java (original)
+++ mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestMapReduceDriver.java Sat Jun 16 16:47:01 2012
@@ -18,19 +18,21 @@
package org.apache.hadoop.mrunit.mapreduce;
import static org.apache.hadoop.mrunit.ExtendedAssert.assertListEquals;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
@@ -408,4 +410,34 @@ public class TestMapReduceDriver {
driver.withOutput(new LongWritable(), new Text("a\t3"));
driver.runTest();
}
+
+ private static class InputPathStoringMapper extends
+ Mapper<Text, LongWritable, Text, LongWritable> {
+ private Path mapInputPath;
+
+ @Override
+ public void map(Text key, LongWritable value, Context context)
+ throws IOException {
+ if (context.getInputSplit() instanceof FileSplit) {
+ mapInputPath = ((FileSplit) context.getInputSplit()).getPath();
+ }
+ }
+
+ private Path getMapInputPath() {
+ return mapInputPath;
+ }
+ }
+
+ @Test
+ public void testMapInputFile() {
+ InputPathStoringMapper mapper = new InputPathStoringMapper();
+ Path mapInputPath = new Path("myfile");
+ driver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
+ driver.setMapInputPath(mapInputPath);
+ assertEquals(mapInputPath.getName(), driver.getMapInputPath().getName());
+ driver.withInput(new Text("a"), new LongWritable(1));
+ driver.runTest();
+ assertNotNull(mapper.getMapInputPath());
+ assertEquals(mapInputPath.getName(), mapper.getMapInputPath().getName());
+ }
}