You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by av...@apache.org on 2012/05/07 18:53:47 UTC
svn commit: r1335190 - in /incubator/hcatalog/branches/branch-0.4:
CHANGES.txt src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Author: avandana
Date: Mon May 7 18:53:46 2012
New Revision: 1335190
URL: http://svn.apache.org/viewvc?rev=1335190&view=rev
Log:
branch 0.4 commit: hcat-403
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Mon May 7 18:53:46 2012
@@ -28,6 +28,8 @@ Release 0.4.1 - Unreleased
HCAT-368 Documentation improvements: doc set & API docs (lefty via gates)
BUG FIXES
+ HCAT-403 MultiOutputFormat will call taskCommit on all formats if only one ever needs commit (rohini via avandana)
+
HCAT-400 HCatalog build fails from git checkout due to missing lib directory (traviscrawford via gates)
Release 0.4.0 - Unreleased
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Mon May 7 18:53:46 2012
@@ -467,7 +467,7 @@ public class MultiOutputFormat extends O
}
}
- private class MultiOutputCommitter extends OutputCommitter {
+ public class MultiOutputCommitter extends OutputCommitter {
private final Map<String, BaseOutputCommitterContainer> outputCommitters;
@@ -516,7 +516,11 @@ public class MultiOutputFormat extends O
public void commitTask(TaskAttemptContext taskContext) throws IOException {
for (String alias : outputCommitters.keySet()) {
BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
- outputContainer.getBaseCommitter().commitTask(outputContainer.getContext());
+ OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
+ TaskAttemptContext committerContext = outputContainer.getContext();
+ if (baseCommitter.needsTaskCommit(committerContext)) {
+ baseCommitter.commitTask(committerContext);
+ }
}
}
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1335190&r1=1335189&r2=1335190&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Mon May 7 18:53:46 2012
@@ -37,8 +37,11 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -157,6 +160,8 @@ public class TestMultiOutputFormat {
configurer.addOutputFormat("out1", TextOutputFormat.class, IntWritable.class, Text.class);
configurer.addOutputFormat("out2", SequenceFileOutputFormat.class, Text.class,
IntWritable.class);
+ configurer.addOutputFormat("out3", NullOutputFormat.class, Text.class,
+ IntWritable.class);
Path outDir = new Path(workDir.getPath(), job.getJobName());
FileOutputFormat.setOutputPath(configurer.getJob("out1"), new Path(outDir, "out1"));
FileOutputFormat.setOutputPath(configurer.getJob("out2"), new Path(outDir, "out2"));
@@ -249,18 +254,40 @@ public class TestMultiOutputFormat {
private static class MultiOutWordCountReducer extends
Reducer<Text, IntWritable, Writable, Writable> {
- private IntWritable result = new IntWritable();
+ private IntWritable count = new IntWritable();
@Override
- protected void reduce(Text key, Iterable<IntWritable> values, Context context)
+ protected void reduce(Text word, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
- result.set(sum);
- MultiOutputFormat.write("out1", result, key, context);
- MultiOutputFormat.write("out2", key, result, context);
+ count.set(sum);
+ MultiOutputFormat.write("out1", count, word, context);
+ MultiOutputFormat.write("out2", word, count, context);
+ MultiOutputFormat.write("out3", word, count, context);
+ }
+ }
+
+ private static class NullOutputFormat<K, V> extends
+ org.apache.hadoop.mapreduce.lib.output.NullOutputFormat<K, V> {
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new OutputCommitter() {
+ public void abortTask(TaskAttemptContext taskContext) { }
+ public void cleanupJob(JobContext jobContext) { }
+ public void commitJob(JobContext jobContext) { }
+ public void commitTask(TaskAttemptContext taskContext) {
+ Assert.fail("needsTaskCommit is false but commitTask was called");
+ }
+ public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+ return false;
+ }
+ public void setupJob(JobContext jobContext) { }
+ public void setupTask(TaskAttemptContext taskContext) { }
+ };
}
}