You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2015/08/05 22:52:31 UTC
hive git commit: HIVE-11456: HCatStorer should honor
mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 96f591446 -> 0f934d29b
HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f934d29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f934d29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f934d29
Branch: refs/heads/master
Commit: 0f934d29bb6bfc992eca050337b8f48fb47aaed8
Parents: 96f5914
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Wed Aug 5 13:50:19 2015 -0700
----------------------------------------------------------------------
.../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ++++---
.../mapreduce/DynamicPartitionFileRecordWriterContainer.java | 3 ++-
.../hive/hcatalog/mapreduce/FileOutputFormatContainer.java | 3 ++-
3 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends OutputFormatContainer {
super(of);
}
- static synchronized String getOutputName(int partition) {
- return "part-" + NUMBER_FORMAT.format(partition);
+ static synchronized String getOutputName(TaskAttemptContext context) {
+ return context.getConfiguration().get("mapreduce.output.basename", "part")
+ + "-" + NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
}
/**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends OutputFormatContainer {
@Override
public RecordWriter<WritableComparable<?>, HCatRecord>
getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
- String name = getOutputName(context.getTaskAttemptID().getTaskID().getId());
+ String name = getOutputName(context);
return new DefaultRecordWriterContainer(context,
getBaseOutputFormat().getRecordWriter(null, new JobConf(context.getConfiguration()), name, InternalUtil.createReporter(context)));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends FileRecordWriterContaine
Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
Path childPath =
- new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));
+ new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+ currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), ""));
RecordWriter baseRecordWriter =
baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),
http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer {
(org.apache.hadoop.mapred.RecordWriter)null, context);
} else {
Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir"));
- Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), "part"));
+ Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()),
+ context.getConfiguration().get("mapreduce.output.basename", "part")));
rw = new StaticPartitionFileRecordWriterContainer(
getBaseOutputFormat().getRecordWriter(