You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2012/09/13 18:21:11 UTC
git commit: CRUNCH-50: Verify MSCRPlanner refactoring fixes issue
with multiple outputs from a single map-only job
Updated Branches:
refs/heads/master 739a4703a -> 7e8d63a20
CRUNCH-50: Verify MSCRPlanner refactoring fixes issue with multiple outputs from a single map-only job
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/7e8d63a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/7e8d63a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/7e8d63a2
Branch: refs/heads/master
Commit: 7e8d63a20d4eb04cff82ff700a06e801074dc372
Parents: 739a470
Author: Josh Wills <jw...@apache.org>
Authored: Thu Sep 13 09:20:48 2012 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Thu Sep 13 09:20:48 2012 -0700
----------------------------------------------------------------------
.../java/org/apache/crunch/MultipleOutputIT.java | 15 +++++++++++++--
1 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/7e8d63a2/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
index d8b5722..0d1f83f 100644
--- a/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
+++ b/crunch/src/it/java/org/apache/crunch/MultipleOutputIT.java
@@ -82,7 +82,16 @@ public class MultipleOutputIT {
run(new MRPipeline(MultipleOutputIT.class, tmpDir.getDefaultConfiguration()), AvroTypeFamily.getInstance());
}
- public void run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
+ @Test
+ public void testParallelDosFused() throws IOException {
+
+ PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, tmpDir.getDefaultConfiguration()), WritableTypeFamily.getInstance());
+
+ // Ensure our multiple outputs were fused into a single job.
+ assertEquals("parallel Dos not fused into a single job", 1, result.getStageResults().size());
+ }
+
+ public PipelineResult run(Pipeline pipeline, PTypeFamily typeFamily) throws IOException {
String inputPath = tmpDir.copyResourceFileName("letters.txt");
String outputPathEven = tmpDir.getFileName("even");
String outputPathOdd = tmpDir.getFileName("odd");
@@ -94,10 +103,12 @@ public class MultipleOutputIT {
pipeline.writeTextFile(evenCountWords, outputPathEven);
pipeline.writeTextFile(oddCountWords, outputPathOdd);
- pipeline.done();
+ PipelineResult result = pipeline.done();
checkFileContents(outputPathEven, Arrays.asList("bb"));
checkFileContents(outputPathOdd, Arrays.asList("a"));
+
+ return result;
}
private void checkFileContents(String filePath, List<String> expected) throws IOException {