You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/03/13 15:06:41 UTC
svn commit: r1300174 -
/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Author: gates
Date: Tue Mar 13 15:06:41 2012
New Revision: 1300174
URL: http://svn.apache.org/viewvc?rev=1300174&view=rev
Log:
HCATALOG-276 Yet more fixes for this.
Modified:
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1300174&r1=1300173&r2=1300174&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java Tue Mar 13 15:06:41 2012
@@ -95,6 +95,9 @@ class FileOutputCommitterContainer exten
@Override
public void abortTask(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
+ // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
+ // Workaround: Set mapred.output.dir from OutputJobInfo.
+ resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().abortTask(HCatMapRedUtil.createTaskAttemptContext(context));
}
}
@@ -114,6 +117,9 @@ class FileOutputCommitterContainer exten
@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
+ // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
+ // Workaround: Set mapred.output.dir from OutputJobInfo.
+ resetMapRedOutputDirFromJobInfo(context.getConfiguration());
return getBaseOutputCommitter().needsTaskCommit(HCatMapRedUtil.createTaskAttemptContext(context));
}else{
// called explicitly through FileRecordWriterContainer.close() if dynamic - return false by default
@@ -126,7 +132,7 @@ class FileOutputCommitterContainer exten
if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
// TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
// Workaround: Set mapred.output.dir from OutputJobInfo.
- context.getConfiguration().set("mapred.output.dir", jobInfo.getLocation());
+ resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().setupJob(HCatMapRedUtil.createJobContext(context));
}
// in dynamic usecase, called through FileRecordWriterContainer
@@ -135,6 +141,9 @@ class FileOutputCommitterContainer exten
@Override
public void setupTask(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
+ // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
+ // Workaround: Set mapred.output.dir from OutputJobInfo.
+ resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().setupTask(HCatMapRedUtil.createTaskAttemptContext(context));
}
}