You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by jd...@apache.org on 2012/04/16 04:10:49 UTC
svn commit: r1326461 - in
/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit:
MapReduceDriver.java MapReduceDriverBase.java mapreduce/MapReduceDriver.java
Author: jdonofrio
Date: Mon Apr 16 02:10:48 2012
New Revision: 1326461
URL: http://svn.apache.org/viewvc?rev=1326461&view=rev
Log:
MRUNIT-99: dont force K2 in MapReduceDriver's to be Comparable since Hadoop dont enforce this on the map output key and value classes
Modified:
incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java
incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
Modified: incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java?rev=1326461&r1=1326460&r2=1326461&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriver.java Mon Apr 16 02:10:48 2012
@@ -47,7 +47,7 @@ import org.apache.hadoop.mrunit.types.Pa
* and before the Reducer.
*/
@SuppressWarnings({ "deprecation", "rawtypes" })
-public class MapReduceDriver<K1, V1, K2 extends Comparable, V2, K3, V3> extends
+public class MapReduceDriver<K1, V1, K2, V2, K3, V3> extends
MapReduceDriverBase<K1, V1, K2, V2, K3, V3> {
public static final Log LOG = LogFactory.getLog(MapReduceDriver.class);
Modified: incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java?rev=1326461&r1=1326460&r2=1326461&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/MapReduceDriverBase.java Mon Apr 16 02:10:48 2012
@@ -45,8 +45,8 @@ import org.apache.hadoop.mrunit.types.Pa
* pair, representing a single unit test.
*/
@SuppressWarnings("rawtypes")
-public abstract class MapReduceDriverBase<K1, V1, K2 extends Comparable, V2, K3, V3>
- extends TestDriver<K1, V1, K3, V3> {
+public abstract class MapReduceDriverBase<K1, V1, K2, V2, K3, V3> extends
+ TestDriver<K1, V1, K3, V3> {
public static final Log LOG = LogFactory.getLog(MapReduceDriverBase.class);
@@ -60,19 +60,6 @@ public abstract class MapReduceDriverBas
public MapReduceDriverBase() {
inputList = new ArrayList<Pair<K1, V1>>();
-
- // create a simple comparator for key grouping and ordering
- final Comparator<K2> simpleComparator = new Comparator<K2>() {
- @SuppressWarnings("unchecked")
- @Override
- public int compare(final K2 o1, final K2 o2) {
- return o1.compareTo(o2);
- }
- };
-
- // assign simple key grouping and ordering comparator
- keyGroupComparator = simpleComparator;
- keyValueOrderComparator = null;
}
/**
@@ -170,6 +157,13 @@ public abstract class MapReduceDriverBas
*/
public List<Pair<K2, List<V2>>> shuffle(final List<Pair<K2, V2>> mapOutputs) {
// step 1 - use the key group comparator to organise map outputs
+ Comparator<K2> keyGroupComparator;
+ if (this.keyGroupComparator == null) {
+ keyGroupComparator = new JobConf(getConfiguration())
+ .getOutputValueGroupingComparator();
+ } else {
+ keyGroupComparator = this.keyGroupComparator;
+ }
final TreeMap<K2, List<Pair<K2, V2>>> groupedByKey = new TreeMap<K2, List<Pair<K2, V2>>>(
keyGroupComparator);
Modified: incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java
URL: http://svn.apache.org/viewvc/incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java?rev=1326461&r1=1326460&r2=1326461&view=diff
==============================================================================
--- incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java (original)
+++ incubator/mrunit/branches/mrunit-0.9/src/main/java/org/apache/hadoop/mrunit/mapreduce/MapReduceDriver.java Mon Apr 16 02:10:48 2012
@@ -45,7 +45,7 @@ import org.apache.hadoop.mrunit.types.Pa
* pair, representing a single unit test.
*/
@SuppressWarnings("rawtypes")
-public class MapReduceDriver<K1, V1, K2 extends Comparable, V2, K3, V3> extends
+public class MapReduceDriver<K1, V1, K2, V2, K3, V3> extends
MapReduceDriverBase<K1, V1, K2, V2, K3, V3> {
public static final Log LOG = LogFactory.getLog(MapReduceDriver.class);