You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by br...@apache.org on 2011/11/28 22:56:24 UTC
svn commit: r1207646 - in /incubator/mrunit/trunk/src:
main/java/org/apache/hadoop/mrunit/
main/java/org/apache/hadoop/mrunit/mapreduce/
test/java/org/apache/hadoop/mrunit/
Author: brock
Date: Mon Nov 28 21:56:19 2011
New Revision: 1207646
URL: http://svn.apache.org/viewvc?rev=1207646&view=rev
Log:
MRUNIT-14: Support JobConfigurable and Closable interfaces
Added:
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestConfigureClose.java
Modified:
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java?rev=1207646&r1=1207645&r2=1207646&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/MapDriver.java Mon Nov 28 21:56:19 2011
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mrunit.mock.MockOutputCollector;
import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -193,9 +194,9 @@ public class MapDriver<K1, V1, K2, V2> e
if (myMapper instanceof Configurable) {
((Configurable)myMapper).setConf(getConfiguration());
}
-
+ myMapper.configure(new JobConf(getConfiguration()));
myMapper.map(inputKey, inputVal, outputCollector, reporter);
-
+ myMapper.close();
return outputCollector.getOutputs();
}
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java?rev=1207646&r1=1207645&r2=1207646&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java Mon Nov 28 21:56:19 2011
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mrunit.mock.MockOutputCollector;
import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -196,9 +197,11 @@ public class ReduceDriver<K1, V1, K2, V2
new MockOutputCollector<K2, V2>(getConfiguration());
MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer, getCounters());
+ myReducer.configure(new JobConf(getConfiguration()));
myReducer.reduce(inputKey, getInputValues().iterator(), outputCollector,
reporter);
-
+ myReducer.close();
+
List<Pair<K2, V2>> outputs = outputCollector.getOutputs();
return outputs;
}
Modified: incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java?rev=1207646&r1=1207645&r2=1207646&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java (original)
+++ incubator/mrunit/trunk/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java Mon Nov 28 21:56:19 2011
@@ -195,7 +195,8 @@ public class MapDriver<K1, V1, K2, V2> e
MockMapContextWrapper<K1, V1, K2, V2> wrapper =
new MockMapContextWrapper<K1, V1, K2, V2>(inputs, getCounters(), getConfiguration());
- myMapper.run(wrapper.getMockContext());
+ Mapper<K1, V1, K2, V2>.Context context = wrapper.getMockContext();
+ myMapper.run(context);
return wrapper.getOutputs();
} catch (InterruptedException ie) {
throw new IOException(ie);
Added: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestConfigureClose.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestConfigureClose.java?rev=1207646&view=auto
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestConfigureClose.java (added)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestConfigureClose.java Mon Nov 28 21:56:19 2011
@@ -0,0 +1,69 @@
+package org.apache.hadoop.mrunit;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Test;
+
+@SuppressWarnings("deprecation")
+public class TestConfigureClose {
+
+ @Test
+ public void testMapperConfigureClose() {
+ final AtomicBoolean configureWasCalled = new AtomicBoolean(false);
+ final AtomicBoolean closeWasCalled = new AtomicBoolean(false);
+ Mapper<Text, Text, Text, Text> mapper = new Mapper<Text, Text, Text, Text>() {
+ @Override
+ public void configure(JobConf arg0) {
+ configureWasCalled.set(true);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeWasCalled.set(true);
+ }
+ @Override
+ public void map(Text key, Text value, OutputCollector<Text, Text> output,
+ Reporter reporter) throws IOException {
+ }
+ };
+ MapDriver<Text, Text, Text, Text> driver = new MapDriver<Text, Text, Text, Text>(mapper);
+ driver.runTest();
+ assertTrue(configureWasCalled.get());
+ assertTrue(closeWasCalled.get());
+ }
+ @Test
+ public void testReducerConfigureClose() {
+ final AtomicBoolean configureWasCalled = new AtomicBoolean(false);
+ final AtomicBoolean closeWasCalled = new AtomicBoolean(false);
+ Reducer<Text, Text, Text, Text> reducer = new Reducer<Text, Text, Text, Text>() {
+ @Override
+ public void configure(JobConf arg0) {
+ configureWasCalled.set(true);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeWasCalled.set(true);
+ }
+
+ @Override
+ public void reduce(Text arg0, Iterator<Text> arg1,
+ OutputCollector<Text, Text> arg2, Reporter arg3) throws IOException {
+ }
+ };
+ ReduceDriver<Text, Text, Text, Text> driver = new ReduceDriver<Text, Text, Text, Text>(reducer);
+ driver.runTest();
+ assertTrue(configureWasCalled.get());
+ assertTrue(closeWasCalled.get());
+ }
+}
Modified: incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java?rev=1207646&r1=1207645&r2=1207646&view=diff
==============================================================================
--- incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java (original)
+++ incubator/mrunit/trunk/src/test/java/org/apache/hadoop/mrunit/TestMapDriver.java Mon Nov 28 21:56:19 2011
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.mrunit;
-import static org.apache.hadoop.mrunit.testutil.ExtendedAssert.*;
+import static org.apache.hadoop.mrunit.testutil.ExtendedAssert.assertListEquals;
import java.io.IOException;
import java.util.ArrayList;
@@ -182,6 +182,6 @@ public class TestMapDriver extends TestC
// expected.
}
}
-
+
}