You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/07/09 15:46:59 UTC
[hive] 03/04: HIVE-21547: Temp Tables: Use stORC format for
temporary tables (Krisztian Kasa via Gopal V)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit cd3d71b34eb482e7abddd68b477ff05644b99d32
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Tue Jul 9 17:36:22 2019 +0200
HIVE-21547: Temp Tables: Use stORC format for temporary tables (Krisztian Kasa via Gopal V)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java | 11 +++++++++++
.../org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java | 3 ++-
.../org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 4 ++--
3 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
index e6b8490..1e8bb22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidOutputFormat.java
@@ -73,6 +73,8 @@ public interface AcidOutputFormat<K extends WritableComparable, V> extends HiveO
* todo: link to AcidUtils?
*/
private long visibilityTxnId = 0;
+ private boolean temporary = false;
+
/**
* Create the options object.
* @param conf Use the given configuration
@@ -334,6 +336,15 @@ public interface AcidOutputFormat<K extends WritableComparable, V> extends HiveO
public long getVisibilityTxnId() {
return visibilityTxnId;
}
+
+ public Options temporary(boolean temporary) {
+ this.temporary = temporary;
+ return this;
+ }
+
+ public boolean isTemporary() {
+ return temporary;
+ }
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
index 2be795b..8980a62 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
@@ -347,7 +347,8 @@ public final class HiveFileFormatUtils {
.inspector(inspector)
.recordIdColumn(rowIdColNum)
.statementId(conf.getStatementId())
- .finalDestination(conf.getDestPath()));
+ .finalDestination(conf.getDestPath())
+ .temporary(conf.isTemporary()));
}
public static <T> T getFromPathRecursively(Map<Path, T> pathToPartitionInfo, Path dir,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 2e4ef31..3fa61d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -355,8 +355,8 @@ public class OrcRecordUpdater implements RecordUpdater {
writerOptions.bufferSize(baseBufferSizeValue / ratio);
writerOptions.stripeSize(baseStripeSizeValue / ratio);
writerOptions.blockPadding(false);
- if (optionsCloneForDelta.getConfiguration().getBoolean(
- HiveConf.ConfVars.HIVE_ORC_DELTA_STREAMING_OPTIMIZATIONS_ENABLED.varname, false)) {
+ if (HiveConf.getBoolVar(optionsCloneForDelta.getConfiguration(),
+ HiveConf.ConfVars.HIVE_ORC_DELTA_STREAMING_OPTIMIZATIONS_ENABLED) || options.isTemporary()) {
writerOptions.encodingStrategy(org.apache.orc.OrcFile.EncodingStrategy.SPEED);
writerOptions.rowIndexStride(0);
writerOptions.getConfiguration().set(OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getAttribute(), "-1.0");