You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by jd...@apache.org on 2012/09/30 18:15:26 UTC
[2/3] git commit: MRUNIT-115: set JobConf specific to the inputformat
reading the results of the outputformat/copying when not using a real
outputformat
MRUNIT-115: set JobConf specific to the inputformat reading the results
of the outputformat/copying when not using a real outputformat
add javadoc, use simpler method name OutputSerializationConfiguration
Project: http://git-wip-us.apache.org/repos/asf/mrunit/repo
Commit: http://git-wip-us.apache.org/repos/asf/mrunit/commit/475c00d6
Tree: http://git-wip-us.apache.org/repos/asf/mrunit/tree/475c00d6
Diff: http://git-wip-us.apache.org/repos/asf/mrunit/diff/475c00d6
Branch: refs/heads/trunk
Commit: 475c00d6db1b3dcf364f3d4325d67ba018afe8d6
Parents: 6a5e5fb
Author: Jim Donofrio <jd...@apache.org>
Authored: Sun Sep 30 12:06:18 2012 -0400
Committer: Jim Donofrio <jd...@apache.org>
Committed: Sun Sep 30 12:06:18 2012 -0400
----------------------------------------------------------------------
.../java/org/apache/hadoop/mrunit/MapDriver.java | 2 +-
.../org/apache/hadoop/mrunit/MapReduceDriver.java | 4 +-
.../org/apache/hadoop/mrunit/ReduceDriver.java | 2 +-
.../java/org/apache/hadoop/mrunit/TestDriver.java | 41 ++++++++++++---
.../mapreduce/AbstractMockContextWrapper.java | 2 +-
.../mrunit/internal/mapreduce/ContextDriver.java | 2 +-
.../hadoop/mrunit/mapreduce/MapReduceDriver.java | 4 +-
7 files changed, 42 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/MapDriver.java b/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
index 8d44d68..9eda69c 100644
--- a/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/MapDriver.java
@@ -130,7 +130,7 @@ public class MapDriver<K1, V1, K2, V2> extends MapDriverBase<K1, V1, K2, V2, Map
initDistributedCache();
final OutputCollectable<K2, V2> outputCollectable = mockOutputCreator
.createOutputCollectable(getConfiguration(),
- getOutputCopyingOrInputFormatConfiguration());
+ getOutputSerializationConfiguration());
final MockReporter reporter = new MockReporter(
MockReporter.ReporterType.Mapper, getCounters(),
getMapInputPath());
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java b/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
index daa72b5..624bb5c 100644
--- a/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
@@ -229,9 +229,9 @@ public class MapReduceDriver<K1, V1, K2, V2, K3, V3>
.newReduceDriver(reducer).withCounters(getCounters())
.withConfiguration(configuration).withAll(inputs);
- if (getOutputCopyingOrInputFormatConfiguration() != null) {
+ if (getOutputSerializationConfiguration() != null) {
reduceDriver
- .withOutputCopyingOrInputFormatConfiguration(getOutputCopyingOrInputFormatConfiguration());
+ .withOutputSerializationConfiguration(getOutputSerializationConfiguration());
}
if (outputFormatClass != null) {
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java b/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
index b76f9d0..d1d53c1 100644
--- a/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/ReduceDriver.java
@@ -133,7 +133,7 @@ public class ReduceDriver<K1, V1, K2, V2> extends
initDistributedCache();
final OutputCollectable<K2, V2> outputCollectable = mockOutputCreator
.createOutputCollectable(getConfiguration(),
- getOutputCopyingOrInputFormatConfiguration());
+ getOutputSerializationConfiguration());
final MockReporter reporter = new MockReporter(
MockReporter.ReporterType.Reducer, getCounters());
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/TestDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/TestDriver.java b/src/main/java/org/apache/hadoop/mrunit/TestDriver.java
index c6f9711..1072f26 100644
--- a/src/main/java/org/apache/hadoop/mrunit/TestDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/TestDriver.java
@@ -51,7 +51,7 @@ public abstract class TestDriver<K1, V1, K2, V2, T extends TestDriver<K1, V1, K2
protected List<Pair<Enum<?>, Long>> expectedEnumCounters;
protected List<Pair<Pair<String, String>, Long>> expectedStringCounters;
protected Configuration configuration;
- private Configuration outputCopyingOrInputFormatConf;
+ private Configuration outputSerializationConfiguration;
private File tmpDistCacheDir;
protected CounterWrapper counterWrapper;
@@ -275,18 +275,45 @@ public abstract class TestDriver<K1, V1, K2, V2, T extends TestDriver<K1, V1, K2
return thisAsTestDriver();
}
- public Configuration getOutputCopyingOrInputFormatConfiguration() {
- return outputCopyingOrInputFormatConf;
+ /**
+ * Get the {@link Configuration} to use when copying output for use with run*
+ * methods or for the InputFormat when reading output back in when setting a
+ * real OutputFormat.
+ *
+ * @return outputSerializationConfiguration, null when no
+ * outputSerializationConfiguration is set
+ */
+ public Configuration getOutputSerializationConfiguration() {
+ return outputSerializationConfiguration;
}
- public void setOutputCopyingOrInputFormatConfiguration(
+ /**
+ * Set the {@link Configuration} to use when copying output for use with run*
+ * methods or for the InputFormat when reading output back in when setting a
+ * real OutputFormat. When this configuration is not set, MRUnit will use the
+ * configuration set with {@link #withConfiguration(Configuration)} or
+ * {@link #setConfiguration(Configuration)}
+ *
+ * @param configuration
+ */
+ public void setOutputSerializationConfiguration(
final Configuration configuration) {
- this.outputCopyingOrInputFormatConf = returnNonNull(configuration);
+ this.outputSerializationConfiguration = returnNonNull(configuration);
}
- public T withOutputCopyingOrInputFormatConfiguration(
+ /**
+ * Set the {@link Configuration} to use when copying output for use with run*
+ * methods or for the InputFormat when reading output back in when setting a
+ * real OutputFormat. When this configuration is not set, MRUnit will use the
+ * configuration set with {@link #withConfiguration(Configuration)} or
+ * {@link #setConfiguration(Configuration)}
+ *
+ * @param configuration
+ * @return this for fluent style
+ */
+ public T withOutputSerializationConfiguration(
Configuration configuration) {
- setOutputCopyingOrInputFormatConfiguration(configuration);
+ setOutputSerializationConfiguration(configuration);
return thisAsTestDriver();
}
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/AbstractMockContextWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/AbstractMockContextWrapper.java b/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/AbstractMockContextWrapper.java
index 786fd65..43d0b53 100644
--- a/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/AbstractMockContextWrapper.java
+++ b/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/AbstractMockContextWrapper.java
@@ -79,7 +79,7 @@ extends TaskInputOutputContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT>> {
try {
if(outputCollectable == null) {
outputCollectable = mockOutputCreator.createOutputCollectable(contextDriver.getConfiguration(),
- contextDriver.getOutputCopyingOrInputFormatConfiguration());
+ contextDriver.getOutputSerializationConfiguration());
}
outputCollectable.collect((KEYOUT)args[0], (VALUEOUT)args[1]);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/ContextDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/ContextDriver.java b/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/ContextDriver.java
index 751d287..5446e95 100644
--- a/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/ContextDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/internal/mapreduce/ContextDriver.java
@@ -24,5 +24,5 @@ public interface ContextDriver {
public Counters getCounters();
public Configuration getConfiguration();
- public Configuration getOutputCopyingOrInputFormatConfiguration();
+ public Configuration getOutputSerializationConfiguration();
}
http://git-wip-us.apache.org/repos/asf/mrunit/blob/475c00d6/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java b/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
index 002e6c5..6af7ac5 100644
--- a/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
+++ b/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
@@ -227,9 +227,9 @@ public class MapReduceDriver<K1, V1, K2, V2, K3, V3> extends
.newReduceDriver(reducer).withCounters(getCounters())
.withConfiguration(configuration).withAll(inputs);
- if (getOutputCopyingOrInputFormatConfiguration() != null) {
+ if (getOutputSerializationConfiguration() != null) {
reduceDriver
- .withOutputCopyingOrInputFormatConfiguration(getOutputCopyingOrInputFormatConfiguration());
+ .withOutputSerializationConfiguration(getOutputSerializationConfiguration());
}
if (outputFormatClass != null) {