You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/08/26 11:42:01 UTC
svn commit: r807942 - in /hadoop/mapreduce/trunk: ./
src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/
src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/
src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/
src/contrib/...
Author: cdouglas
Date: Wed Aug 26 09:42:00 2009
New Revision: 807942
URL: http://svn.apache.org/viewvc?rev=807942&view=rev
Log:
MAPREDUCE-910. Support counters in MRUnit. Contributed by Aaron Kimball
Added:
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Aug 26 09:42:00 2009
@@ -269,6 +269,8 @@
MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests
in local mode. (Aaron Kimball via tomwhite)
+ MAPREDUCE-910. Support counters in MRUnit. (Aaron Kimball via cdouglas)
+
BUG FIXES
MAPREDUCE-878. Rename fair scheduler design doc to
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapDriver.java Wed Aug 26 09:42:00 2009
@@ -24,6 +24,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mrunit.mock.MockOutputCollector;
import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -43,12 +44,33 @@
public static final Log LOG = LogFactory.getLog(MapDriver.class);
private Mapper<K1, V1, K2, V2> myMapper;
+ private Counters counters;
public MapDriver(final Mapper<K1, V1, K2, V2> m) {
myMapper = m;
+ counters = new Counters();
}
public MapDriver() {
+ counters = new Counters();
+ }
+
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public MapDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
}
/**
@@ -165,7 +187,7 @@
public List<Pair<K2, V2>> run() throws IOException {
MockOutputCollector<K2, V2> outputCollector =
new MockOutputCollector<K2, V2>();
- MockReporter reporter = new MockReporter(MockReporter.ReporterType.Mapper);
+ MockReporter reporter = new MockReporter(MockReporter.ReporterType.Mapper, getCounters());
myMapper.map(inputKey, inputVal, outputCollector, reporter);
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/MapReduceDriver.java Wed Aug 26 09:42:00 2009
@@ -29,6 +29,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mrunit.types.Pair;
@@ -55,11 +56,13 @@
private Mapper<K1, V1, K2, V2> myMapper;
private Reducer<K2, V2, K3, V3> myReducer;
private Reducer<K2, V2, K2, V2> myCombiner;
+ private Counters counters;
public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
final Reducer<K2, V2, K3, V3> r) {
myMapper = m;
myReducer = r;
+ counters = new Counters();
}
public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
@@ -68,9 +71,29 @@
myMapper = m;
myReducer = r;
myCombiner = c;
+ counters = new Counters();
}
public MapReduceDriver() {
+ counters = new Counters();
+ }
+
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
}
/** Set the Mapper instance to use with this test driver
@@ -227,7 +250,10 @@
+ sb.toString() + ")");
reduceOutputs.addAll(new ReduceDriver<K2, V2, OUTKEY, OUTVAL>(reducer)
- .withInputKey(inputKey).withInputValues(inputValues).run());
+ .withCounters(getCounters())
+ .withInputKey(inputKey)
+ .withInputValues(inputValues)
+ .run());
}
return reduceOutputs;
@@ -243,7 +269,7 @@
LOG.debug("Mapping input " + input.toString() + ")");
mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput(
- input).run());
+ input).withCounters(getCounters()).run());
}
if (myCombiner != null) {
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/PipelineMapReduceDriver.java Wed Aug 26 09:42:00 2009
@@ -25,6 +25,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mrunit.types.Pair;
@@ -56,15 +57,18 @@
private List<Pair<Mapper, Reducer>> mapReducePipeline;
private List<Pair<K1, V1>> inputList;
+ private Counters counters;
public PipelineMapReduceDriver(final List<Pair<Mapper, Reducer>> pipeline) {
this.mapReducePipeline = copyMapReduceList(pipeline);
this.inputList = new ArrayList<Pair<K1, V1>>();
+ this.counters = new Counters();
}
public PipelineMapReduceDriver() {
this.mapReducePipeline = new ArrayList<Pair<Mapper, Reducer>>();
this.inputList = new ArrayList<Pair<K1, V1>>();
+ this.counters = new Counters();
}
private List<Pair<Mapper, Reducer>> copyMapReduceList(List<Pair<Mapper, Reducer>> lst) {
@@ -77,6 +81,25 @@
return outList;
}
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public PipelineMapReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
+ }
+
+
/** Add a Mapper and Reducer instance to the pipeline to use with this test driver
* @param m The Mapper instance to add to the pipeline
* @param r The Reducer instance to add to the pipeline
@@ -282,6 +305,8 @@
// Create a MapReduceDriver to run this phase of the pipeline.
MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());
+ mrDriver.setCounters(getCounters());
+
// Add the inputs from the user, or from the previous stage of the pipeline.
for (Object input : inputs) {
mrDriver.addInput((Pair) input);
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/ReduceDriver.java Wed Aug 26 09:42:00 2009
@@ -25,6 +25,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mrunit.mock.MockOutputCollector;
import org.apache.hadoop.mrunit.mock.MockReporter;
@@ -45,14 +46,36 @@
public static final Log LOG = LogFactory.getLog(ReduceDriver.class);
private Reducer<K1, V1, K2, V2> myReducer;
+ private Counters counters;
public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
myReducer = r;
+ counters = new Counters();
}
public ReduceDriver() {
+ counters = new Counters();
}
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public ReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
+ }
+
+
/**
* Sets the reducer object to use for this test
*
@@ -172,7 +195,7 @@
public List<Pair<K2, V2>> run() throws IOException {
MockOutputCollector<K2, V2> outputCollector =
new MockOutputCollector<K2, V2>();
- MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer);
+ MockReporter reporter = new MockReporter(MockReporter.ReporterType.Reducer, getCounters());
myReducer.reduce(inputKey, inputValues.iterator(), outputCollector,
reporter);
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapDriver.java Wed Aug 26 09:42:00 2009
@@ -26,6 +26,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mrunit.MapDriverBase;
import org.apache.hadoop.mrunit.mapreduce.mock.MockMapContextWrapper;
@@ -45,12 +46,15 @@
public static final Log LOG = LogFactory.getLog(MapDriver.class);
private Mapper<K1, V1, K2, V2> myMapper;
+ private Counters counters;
public MapDriver(final Mapper<K1, V1, K2, V2> m) {
myMapper = m;
+ counters = new Counters();
}
public MapDriver() {
+ counters = new Counters();
}
@@ -76,6 +80,24 @@
return myMapper;
}
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public MapDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
+ }
+
/**
* Identical to setInputKey() but with fluent programming style
*
@@ -172,7 +194,7 @@
try {
MockMapContextWrapper<K1, V1, K2, V2> wrapper = new MockMapContextWrapper();
MockMapContextWrapper<K1, V1, K2, V2>.MockMapContext context =
- wrapper.getMockContext(inputs);
+ wrapper.getMockContext(inputs, getCounters());
myMapper.run(context);
return context.getOutputs();
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Wed Aug 26 09:42:00 2009
@@ -29,6 +29,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mrunit.MapReduceDriverBase;
@@ -51,14 +52,17 @@
private Mapper<K1, V1, K2, V2> myMapper;
private Reducer<K2, V2, K3, V3> myReducer;
+ private Counters counters;
public MapReduceDriver(final Mapper<K1, V1, K2, V2> m,
final Reducer<K2, V2, K3, V3> r) {
myMapper = m;
myReducer = r;
+ counters = new Counters();
}
public MapReduceDriver() {
+ counters = new Counters();
}
/** Set the Mapper instance to use with this test driver
@@ -107,6 +111,24 @@
return myReducer;
}
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public MapReduceDriver<K1, V1, K2, V2, K3, V3> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
+ }
+
/**
* Identical to addInput() but returns self for fluent programming style
* @param key
@@ -180,7 +202,7 @@
LOG.debug("Mapping input " + input.toString() + ")");
mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput(
- input).run());
+ input).withCounters(getCounters()).run());
}
List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs);
@@ -195,6 +217,7 @@
+ sb.toString() + ")");
reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer)
+ .withCounters(getCounters())
.withInputKey(inputKey).withInputValues(inputValues).run());
}
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/ReduceDriver.java Wed Aug 26 09:42:00 2009
@@ -26,6 +26,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mrunit.ReduceDriverBase;
import org.apache.hadoop.mrunit.mapreduce.mock.MockReduceContextWrapper;
@@ -46,12 +47,15 @@
public static final Log LOG = LogFactory.getLog(ReduceDriver.class);
private Reducer<K1, V1, K2, V2> myReducer;
+ private Counters counters;
public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
myReducer = r;
+ counters = new Counters();
}
public ReduceDriver() {
+ counters = new Counters();
}
/**
@@ -80,6 +84,24 @@
return myReducer;
}
+ /** @return the counters used in this test */
+ public Counters getCounters() {
+ return counters;
+ }
+
+ /** Sets the counters object to use for this test.
+ * @param ctrs The counters object to use.
+ */
+ public void setCounters(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ /** Sets the counters to use and returns self for fluent style */
+ public ReduceDriver<K1, V1, K2, V2> withCounters(final Counters ctrs) {
+ setCounters(ctrs);
+ return this;
+ }
+
/**
* Identical to setInputKey() but with fluent programming style
*
@@ -177,7 +199,7 @@
try {
MockReduceContextWrapper<K1, V1, K2, V2> wrapper = new MockReduceContextWrapper();
MockReduceContextWrapper<K1, V1, K2, V2>.MockReduceContext context =
- wrapper.getMockContext(inputs);
+ wrapper.getMockContext(inputs, getCounters());
myReducer.run(context);
return context.getOutputs();
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockMapContextWrapper.java Wed Aug 26 09:42:00 2009
@@ -21,7 +21,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -57,12 +57,12 @@
private Pair<KEYIN, VALUEIN> curInput;
private MockOutputCollector<KEYOUT, VALUEOUT> output;
- public MockMapContext(final List<Pair<KEYIN, VALUEIN>> in)
+ public MockMapContext(final List<Pair<KEYIN, VALUEIN>> in, final Counters counters)
throws IOException, InterruptedException {
super(new Configuration(),
new TaskAttemptID("mrunit-jt", 0, TaskType.MAP, 0, 0),
- null, null, new MockOutputCommitter(), null, null);
+ null, null, new MockOutputCommitter(), new MockReporter(counters), null);
this.inputIter = in.iterator();
this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
}
@@ -96,17 +96,6 @@
output.collect(key, value);
}
- /** This method does nothing in the mock version. */
- public Counter getCounter(Enum<?> counterName) {
- return null;
- }
-
- @Override
- /** This method does nothing in the mock version. */
- public Counter getCounter(String groupName, String counterName) {
- return null;
- }
-
@Override
/** This method does nothing in the mock version. */
public void progress() {
@@ -126,9 +115,9 @@
}
}
- public MockMapContext getMockContext(List<Pair<KEYIN, VALUEIN>> inputs)
+ public MockMapContext getMockContext(List<Pair<KEYIN, VALUEIN>> inputs, Counters counters)
throws IOException, InterruptedException {
- return new MockMapContext(inputs);
+ return new MockMapContext(inputs, counters);
}
}
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReduceContextWrapper.java Wed Aug 26 09:42:00 2009
@@ -22,7 +22,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.ReduceContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -63,13 +63,13 @@
private MockOutputCollector<KEYOUT, VALUEOUT> output;
- public MockReduceContext(final List<Pair<KEYIN, List<VALUEIN>>> in)
+ public MockReduceContext(final List<Pair<KEYIN, List<VALUEIN>>> in, final Counters counters)
throws IOException, InterruptedException {
super(new Configuration(),
new TaskAttemptID("mrunit-jt", 0, TaskType.REDUCE, 0, 0),
new MockRawKeyValueIterator(), null, null,
- new MockOutputCommitter(), null, null,
+ new MockOutputCommitter(), new MockReporter(counters), null,
(Class) Text.class, (Class) Text.class);
this.inputIter = in.iterator();
this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
@@ -159,17 +159,6 @@
output.collect(key, value);
}
- /** This method does nothing in the mock version. */
- public Counter getCounter(Enum<?> counterName) {
- return null;
- }
-
- @Override
- /** This method does nothing in the mock version. */
- public Counter getCounter(String groupName, String counterName) {
- return null;
- }
-
@Override
/** This method does nothing in the mock version. */
public void progress() {
@@ -189,9 +178,10 @@
}
}
- public MockReduceContext getMockContext(List<Pair<KEYIN, List<VALUEIN>>> inputs)
+ public MockReduceContext getMockContext(List<Pair<KEYIN, List<VALUEIN>>> inputs,
+ Counters counters)
throws IOException, InterruptedException {
- return new MockReduceContext(inputs);
+ return new MockReduceContext(inputs, counters);
}
}
Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java (added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mapreduce/mock/MockReporter.java Wed Aug 26 09:42:00 2009
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.mapreduce.mock;
+
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.StatusReporter;
+
+public class MockReporter extends StatusReporter {
+
+ private Counters counters;
+
+ public MockReporter(final Counters ctrs) {
+ this.counters = ctrs;
+ }
+
+ @Override
+ public void setStatus(String status) {
+ // do nothing.
+ }
+
+ @Override
+ public void progress() {
+ // do nothing.
+ }
+
+ @Override
+ public Counter getCounter(String group, String name) {
+ Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(group, name);
+ }
+
+ return counter;
+ }
+
+ @Override
+ public Counter getCounter(Enum key) {
+ Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(key);
+ }
+
+ return counter;
+ }
+}
+
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/java/org/apache/hadoop/mrunit/mock/MockReporter.java Wed Aug 26 09:42:00 2009
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.mrunit.mock;
+import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.Counters.Counter;
@@ -24,6 +25,7 @@
public class MockReporter implements Reporter {
private MockInputSplit inputSplit = new MockInputSplit();
+ private Counters counters;
public enum ReporterType {
Mapper,
@@ -32,8 +34,9 @@
private ReporterType typ;
- public MockReporter(final ReporterType kind) {
+ public MockReporter(final ReporterType kind, final Counters ctrs) {
this.typ = kind;
+ this.counters = ctrs;
}
@Override
@@ -48,12 +51,16 @@
@Override
public void incrCounter(Enum key, long amount) {
- // do nothing.
+ if (null != counters) {
+ counters.incrCounter(key, amount);
+ }
}
@Override
public void incrCounter(String group, String counter, long amount) {
- // do nothing.
+ if (null != counters) {
+ counters.incrCounter(group, counter, amount);
+ }
}
@Override
@@ -67,15 +74,23 @@
}
@Override
- public Counter getCounter(String s1, String s2) {
- // do nothing
- return null;
+ public Counter getCounter(String group, String name) {
+ Counters.Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(group, name);
+ }
+
+ return counter;
}
@Override
public Counter getCounter(Enum key) {
- // do nothing
- return null;
+ Counters.Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(key);
+ }
+
+ return counter;
}
}
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/AllTests.java Wed Aug 26 09:42:00 2009
@@ -42,6 +42,7 @@
suite.addTestSuite(TestReduceDriver.class);
suite.addTestSuite(TestTestDriver.class);
suite.addTestSuite(TestExample.class);
+ suite.addTestSuite(TestCounters.class);
suite.addTest(org.apache.hadoop.mrunit.types.AllTests.suite());
suite.addTest(org.apache.hadoop.mrunit.mapreduce.AllTests.suite());
Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java (added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/TestCounters.java Wed Aug 26 09:42:00 2009
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test counters usage in various drivers.
+ */
+public class TestCounters extends TestCase {
+
+ private static final String GROUP = "GROUP";
+ private static final String ELEM = "ELEM";
+
+ private class CounterMapper extends MapReduceBase implements Mapper<Text, Text, Text, Text> {
+ public void map(Text k, Text v, OutputCollector<Text, Text> out, Reporter r)
+ throws IOException {
+
+ r.incrCounter(GROUP, ELEM, 1);
+
+ // Emit the same (k, v) pair twice.
+ out.collect(k, v);
+ out.collect(k, v);
+ }
+ }
+
+ private class CounterReducer extends MapReduceBase implements Reducer<Text, Text, Text, Text> {
+ public void reduce(Text k, Iterator<Text> vals, OutputCollector<Text, Text> out, Reporter r)
+ throws IOException {
+
+ while (vals.hasNext()) {
+ r.incrCounter(GROUP, ELEM, 1);
+ out.collect(k, vals.next());
+ }
+ }
+ }
+
+ @Test
+ public void testMapper() throws IOException {
+ Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+ MapDriver<Text, Text, Text, Text> driver = new MapDriver<Text, Text, Text, Text>(mapper);
+ driver.withInput(new Text("foo"), new Text("bar")).run();
+ assertEquals("Expected 1 counter increment", 1,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+
+ @Test
+ public void testReducer() throws IOException {
+ Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+ ReduceDriver<Text, Text, Text, Text> driver = new ReduceDriver<Text, Text, Text, Text>(reducer);
+ driver.withInputKey(new Text("foo"))
+ .withInputValue(new Text("bar"))
+ .run();
+ assertEquals("Expected 1 counter increment", 1,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+
+ @Test
+ public void testMapReduce() throws IOException {
+ Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+ Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+ MapReduceDriver<Text, Text, Text, Text, Text, Text> driver =
+ new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
+
+ driver.withInput(new Text("foo"), new Text("bar"))
+ .run();
+
+ assertEquals("Expected counter=3", 3,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+
+ @Test
+ public void testPipeline() throws IOException {
+ Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+ Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+ PipelineMapReduceDriver<Text, Text, Text, Text> driver =
+ new PipelineMapReduceDriver<Text, Text, Text, Text>();
+
+ driver.withMapReduce(mapper, reducer)
+ .withMapReduce(mapper, reducer)
+ .withInput(new Text("foo"), new Text("bar"))
+ .run();
+
+ assertEquals("Expected counter=9", 9,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+}
+
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/AllTests.java Wed Aug 26 09:42:00 2009
@@ -34,6 +34,7 @@
suite.addTestSuite(TestMapDriver.class);
suite.addTestSuite(TestReduceDriver.class);
suite.addTestSuite(TestMapReduceDriver.class);
+ suite.addTestSuite(TestCounters.class);
return suite;
}
Added: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java?rev=807942&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java (added)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mapreduce/TestCounters.java Wed Aug 26 09:42:00 2009
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mrunit.mapreduce;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test counters usage in various drivers.
+ */
+public class TestCounters extends TestCase {
+
+ private static final String GROUP = "GROUP";
+ private static final String ELEM = "ELEM";
+
+ private class CounterMapper extends Mapper<Text, Text, Text, Text> {
+ public void map(Text k, Text v, Context context)
+ throws IOException, InterruptedException {
+
+ context.getCounter(GROUP, ELEM).increment(1);
+
+ // Emit the same (k, v) pair twice.
+ context.write(k, v);
+ context.write(k, v);
+ }
+ }
+
+ private class CounterReducer extends Reducer<Text, Text, Text, Text> {
+ public void reduce(Text k, Iterable<Text> vals, Context context)
+ throws IOException, InterruptedException {
+
+ for(Text val : vals) {
+ context.getCounter(GROUP, ELEM).increment(1);
+ context.write(k, val);
+ }
+ }
+ }
+
+ @Test
+ public void testMapper() throws IOException {
+ Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+ MapDriver<Text, Text, Text, Text> driver = new MapDriver<Text, Text, Text, Text>(mapper);
+ driver.withInput(new Text("foo"), new Text("bar")).run();
+ assertEquals("Expected 1 counter increment", 1,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+
+ @Test
+ public void testReducer() throws IOException {
+ Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+ ReduceDriver<Text, Text, Text, Text> driver = new ReduceDriver<Text, Text, Text, Text>(reducer);
+ driver.withInputKey(new Text("foo"))
+ .withInputValue(new Text("bar"))
+ .run();
+ assertEquals("Expected 1 counter increment", 1,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+
+ @Test
+ public void testMapReduce() throws IOException {
+ Mapper<Text, Text, Text, Text> mapper = new CounterMapper();
+ Reducer<Text, Text, Text, Text> reducer = new CounterReducer();
+ MapReduceDriver<Text, Text, Text, Text, Text, Text> driver =
+ new MapReduceDriver<Text, Text, Text, Text, Text, Text>(mapper, reducer);
+
+ driver.withInput(new Text("foo"), new Text("bar"))
+ .run();
+
+ assertEquals("Expected counter=3", 3,
+ driver.getCounters().findCounter(GROUP, ELEM).getValue());
+ }
+}
+
Modified: hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java?rev=807942&r1=807941&r2=807942&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mrunit/src/test/org/apache/hadoop/mrunit/mock/TestMockReporter.java Wed Aug 26 09:42:00 2009
@@ -27,7 +27,7 @@
@Test
public void testGetInputSplitForMapper() {
- InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper).getInputSplit();
+ InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper, null).getInputSplit();
assertTrue(null != split);
}
@@ -36,7 +36,7 @@
@Test
public void testGetInputSplitForReducer() {
try {
- new MockReporter(MockReporter.ReporterType.Reducer).getInputSplit();
+ new MockReporter(MockReporter.ReporterType.Reducer, null).getInputSplit();
fail(); // shouldn't get here
} catch (UnsupportedOperationException uoe) {
// expected this.