You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by to...@apache.org on 2012/04/23 16:44:29 UTC
svn commit: r1329321 - in /incubator/hcatalog/trunk: CHANGES.txt build.xml
src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Author: toffer
Date: Mon Apr 23 16:44:29 2012
New Revision: 1329321
URL: http://svn.apache.org/viewvc?rev=1329321&view=rev
Log:
HCATALOG-379 Fix mapred.out.dir hack in FileOutputCommitterContainer (toffer)
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/build.xml
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1329321&r1=1329320&r2=1329321&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Mon Apr 23 16:44:29 2012
@@ -85,6 +85,8 @@ Release 0.4.0 - Unreleased
HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
IMPROVEMENTS
+ HCAT-379 Fix mapred.out.dir hack in FileOutputCommitterContainer (toffer)
+
HCAT-69 Fix token usage from HCat (toffer)
HCAT-352 Need tests for complex types (gates)
Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1329321&r1=1329320&r2=1329321&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Mon Apr 23 16:44:29 2012
@@ -69,7 +69,7 @@
<property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
<property name="test.output" value="no"/>
<property name="test.warehouse.dir" value="/tmp/hcat_junit_warehouse"/>
- <property name="test.excludes" value="${test.src.dir}/e2e/**"/>
+ <property name="test.excludes" value="e2e/**"/>
<!-- e2e test properties -->
<property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1329321&r1=1329320&r2=1329321&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java Mon Apr 23 16:44:29 2012
@@ -95,9 +95,6 @@ class FileOutputCommitterContainer exten
@Override
public void abortTask(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().abortTask(HCatMapRedUtil.createTaskAttemptContext(context));
}
}
@@ -105,11 +102,6 @@ class FileOutputCommitterContainer exten
@Override
public void commitTask(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
- //TODO fix this hack, something wrong with pig
- //running multiple storers in a single job, the real output dir got overwritten or something
- //the location in OutputJobInfo is still correct so we'll use that
- //TestHCatStorer.testMultiPartColsInData() used to fail without this
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().commitTask(HCatMapRedUtil.createTaskAttemptContext(context));
}
}
@@ -117,9 +109,6 @@ class FileOutputCommitterContainer exten
@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
return getBaseOutputCommitter().needsTaskCommit(HCatMapRedUtil.createTaskAttemptContext(context));
}else{
// called explicitly through FileRecordWriterContainer.close() if dynamic - return false by default
@@ -130,9 +119,6 @@ class FileOutputCommitterContainer exten
@Override
public void setupJob(JobContext context) throws IOException {
if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().setupJob(HCatMapRedUtil.createJobContext(context));
}
// in dynamic usecase, called through FileRecordWriterContainer
@@ -141,9 +127,6 @@ class FileOutputCommitterContainer exten
@Override
public void setupTask(TaskAttemptContext context) throws IOException {
if (!dynamicPartitioningUsed){
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().setupTask(HCatMapRedUtil.createTaskAttemptContext(context));
}
}
@@ -157,9 +140,6 @@ class FileOutputCommitterContainer exten
}
if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(mapRedJobContext.getConfiguration());
getBaseOutputCommitter().abortJob(mapRedJobContext, state);
}
else if (dynamicPartitioningUsed){
@@ -225,9 +205,6 @@ class FileOutputCommitterContainer exten
discoverPartitions(jobContext);
}
if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(jobContext.getConfiguration());
getBaseOutputCommitter().commitJob(HCatMapRedUtil.createJobContext(jobContext));
}
// create _SUCCESS FILE if so requested.
@@ -265,9 +242,6 @@ class FileOutputCommitterContainer exten
if( table.getPartitionKeys().size() == 0 ) {
//non partitioned table
if(getBaseOutputCommitter() != null && !dynamicPartitioningUsed) {
- // TODO: Hack! Pig messes up mapred.output.dir, when 2 Storers are used in the same Pig script.
- // Workaround: Set mapred.output.dir from OutputJobInfo.
- resetMapRedOutputDirFromJobInfo(context.getConfiguration());
getBaseOutputCommitter().cleanupJob(HCatMapRedUtil.createJobContext(context));
}
else if (dynamicPartitioningUsed){
@@ -683,15 +657,4 @@ class FileOutputCommitterContainer exten
}
}
- /**
- * TODO: Clean up this Hack! Resetting mapred.output.dir from OutputJobInfo.
- * This works around PIG-2578, where Pig messes up output-directory
- * if multiple storers are used in the same pig-script.
- * @param config The configuration whose mapred.output.dir is to be reset.
- */
- private void resetMapRedOutputDirFromJobInfo(Configuration config) {
- String outputLocation = jobInfo.getLocation();
- if (outputLocation != null)
- config.set("mapred.output.dir", outputLocation);
- }
}