You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrunit.apache.org by db...@apache.org on 2012/12/04 14:16:12 UTC
git commit: MRUNIT-165: MapReduceDriver calls Mapper#cleanup for each
input instead of once (unit test contributed by Yoni Ben-Meshulam)
Updated Branches:
refs/heads/trunk 2f9b6f787 -> 92ad229da
MRUNIT-165: MapReduceDriver calls Mapper#cleanup for each input instead of once (unit test contributed by Yoni Ben-Meshulam)
Project: http://git-wip-us.apache.org/repos/asf/mrunit/repo
Commit: http://git-wip-us.apache.org/repos/asf/mrunit/commit/92ad229d
Tree: http://git-wip-us.apache.org/repos/asf/mrunit/tree/92ad229d
Diff: http://git-wip-us.apache.org/repos/asf/mrunit/diff/92ad229d
Branch: refs/heads/trunk
Commit: 92ad229dafea47db18fa978e809d13261721a806
Parents: 2f9b6f7
Author: Dave Beech <db...@apache.org>
Authored: Tue Dec 4 13:15:27 2012 +0000
Committer: Dave Beech <db...@apache.org>
Committed: Tue Dec 4 13:15:27 2012 +0000
----------------------------------------------------------------------
.../hadoop/mrunit/mapreduce/StatefulMapper.java | 61 +++++++++++++++
.../mrunit/mapreduce/TestStatefulMapReduce.java | 47 +++++++++++
2 files changed, 108 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mrunit/blob/92ad229d/src/test/java/org/apache/hadoop/mrunit/mapreduce/StatefulMapper.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/mrunit/mapreduce/StatefulMapper.java b/src/test/java/org/apache/hadoop/mrunit/mapreduce/StatefulMapper.java
new file mode 100644
index 0000000..1a968ef
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/mrunit/mapreduce/StatefulMapper.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.mapreduce;
+
+import java.io.IOException;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+/**
+ * A Mapper implementation which maintains some state in the form of a counter.
+ */
+public class StatefulMapper extends
+ Mapper<LongWritable, Text, Text, IntWritable> {
+
+ public static final Text KEY = new Text("SomeKey");
+ private Integer someState = 0;
+
+ /**
+ * Increment someState for each input.
+ *
+ * @param context
+ * the Hadoop job Map context
+ * @throws java.io.IOException
+ */
+ @Override
+ public void map(LongWritable key, Text value, Context context)
+ throws IOException, InterruptedException {
+ this.someState++;
+ }
+
+ /**
+ * Runs once after all maps have occurred. Dumps the accumulated state to the
+ * output.
+ *
+ * @param context
+ * the Hadoop job Map context
+ */
+ @Override
+ protected void cleanup(Context context) throws IOException,
+ InterruptedException {
+ context.write(KEY, new IntWritable(this.someState));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mrunit/blob/92ad229d/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestStatefulMapReduce.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestStatefulMapReduce.java b/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestStatefulMapReduce.java
new file mode 100644
index 0000000..ad356b8
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/mrunit/mapreduce/TestStatefulMapReduce.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mrunit.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.junit.Test;
+
+/**
+ * Test that the mapper state is maintained. Essentially checks that the cleanup
+ * method is called only once per task.
+ */
+public class TestStatefulMapReduce {
+
+ @Test
+ public void testClosedFormMapReduce() throws IOException {
+
+ MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver
+ = MapReduceDriver.newMapReduceDriver(new StatefulMapper(), new Reducer());
+
+ mapReduceDriver.addInput(new LongWritable(1L), new Text("hello"));
+ mapReduceDriver.addInput(new LongWritable(2L), new Text("schmo"));
+ mapReduceDriver.withOutput(new Text("SomeKey"), new IntWritable(2));
+ mapReduceDriver.runTest();
+
+ }
+
+}