You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2012/03/09 22:13:42 UTC
svn commit: r1299047 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/ja...
Author: vinodkv
Date: Fri Mar 9 21:13:42 2012
New Revision: 1299047
URL: http://svn.apache.org/viewvc?rev=1299047&view=rev
Log:
MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job' whose tasks don't write any outputs. Contributed by Robert Joseph Evans.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Mar 9 21:13:42 2012
@@ -304,6 +304,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3975. Default value not set for Configuration parameter
mapreduce.job.local.dir (Eric Payne via bobby)
+ MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job'
+ whose tasks don't write any outputs. (Robert Joseph Evans via vinodkv)
+
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java Fri Mar 9 21:13:42 2012
@@ -278,11 +278,11 @@ public class FileOutputCommitter extends
*/
public void setupJob(JobContext context) throws IOException {
if (hasOutputPath()) {
- Path pendingJobAttemptsPath = getPendingJobAttemptsPath();
- FileSystem fs = pendingJobAttemptsPath.getFileSystem(
+ Path jobAttemptPath = getJobAttemptPath(context);
+ FileSystem fs = jobAttemptPath.getFileSystem(
context.getConfiguration());
- if (!fs.mkdirs(pendingJobAttemptsPath)) {
- LOG.error("Mkdirs failed to create " + pendingJobAttemptsPath);
+ if (!fs.mkdirs(jobAttemptPath)) {
+ LOG.error("Mkdirs failed to create " + jobAttemptPath);
}
} else {
LOG.warn("Output Path is null in setupJob()");
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java?rev=1299047&r1=1299046&r2=1299047&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java Fri Mar 9 21:13:42 2012
@@ -122,6 +122,28 @@ public class TestFileOutputCommitter ext
assertEquals(output, expectedOutput.toString());
FileUtil.fullyDelete(new File(outDir.toString()));
}
+
+ public void testEmptyOutput() throws Exception {
+ Job job = Job.getInstance();
+ FileOutputFormat.setOutputPath(job, outDir);
+ Configuration conf = job.getConfiguration();
+ conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
+ JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
+ TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
+ FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
+
+ // setup
+ committer.setupJob(jContext);
+ committer.setupTask(tContext);
+
+ // Do not write any output
+
+ // do commit
+ committer.commitTask(tContext);
+ committer.commitJob(jContext);
+
+ FileUtil.fullyDelete(new File(outDir.toString()));
+ }
@SuppressWarnings("unchecked")
public void testAbort() throws IOException, InterruptedException {