You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2016/06/09 17:12:34 UTC
hive git commit: HIVE-13563 : Hive Streaming does not honor
orc.compress.size and orc.stripe.size table properties (Wei Zheng,
reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master c9c0c5d1d -> f67c862e3
HIVE-13563 : Hive Streaming does not honor orc.compress.size and orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f67c862e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f67c862e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f67c862e
Branch: refs/heads/master
Commit: f67c862e30cc8c3d230853c688c5c70d30d207f3
Parents: c9c0c5d
Author: Wei Zheng <we...@apache.org>
Authored: Thu Jun 9 10:12:19 2016 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Thu Jun 9 10:12:19 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
orc/src/java/org/apache/orc/OrcConf.java | 2 +
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 42 +++++++++++++++-----
.../hive/ql/io/orc/TestOrcRecordUpdater.java | 4 ++
.../clientpositive/tez/acid_globallimit.q.out | 6 +--
5 files changed, 42 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bb0ca3a..fe69ffa 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1206,6 +1206,8 @@ public class HiveConf extends Configuration {
"to use dictionary or not will be retained thereafter."),
HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 * 1024,
"Define the default ORC buffer size, in bytes."),
+ HIVE_ORC_BASE_DELTA_RATIO("hive.exec.orc.base.delta.ratio", 8, "The ratio of base writer and\n" +
+ "delta writer in terms of STRIPE_SIZE and BUFFER_SIZE."),
HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true,
"Define the default block padding, which pads stripes to the HDFS block boundaries."),
HIVE_ORC_BLOCK_PADDING_TOLERANCE("hive.exec.orc.block.padding.tolerance", 0.05f,
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/orc/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcConf.java b/orc/src/java/org/apache/orc/OrcConf.java
index 6fcbb72..357318d 100644
--- a/orc/src/java/org/apache/orc/OrcConf.java
+++ b/orc/src/java/org/apache/orc/OrcConf.java
@@ -40,6 +40,8 @@ public enum OrcConf {
" number of rows n index entry represents.)"),
BUFFER_SIZE("orc.compress.size", "hive.exec.orc.default.buffer.size",
256 * 1024, "Define the default ORC buffer size, in bytes."),
+ BASE_DELTA_RATIO("orc.base.delta.ratio", "hive.exec.orc.base.delta.ratio", 8,
+ "The ratio of base writer and delta writer in terms of STRIPE_SIZE and BUFFER_SIZE."),
BLOCK_PADDING("orc.block.padding", "hive.exec.orc.default.block.padding",
true,
"Define whether stripes should be padded to the HDFS block boundaries."),
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 4bf2403..e577961 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.OrcConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -204,19 +205,38 @@ public class OrcRecordUpdater implements RecordUpdater {
flushLengths = null;
}
OrcFile.WriterOptions writerOptions = null;
- if (options instanceof OrcOptions) {
- writerOptions = ((OrcOptions) options).getOrcOptions();
- }
- if (writerOptions == null) {
- writerOptions = OrcFile.writerOptions(options.getTableProperties(),
- options.getConfiguration());
- }
- writerOptions.fileSystem(fs).callback(indexBuilder);
- if (!options.isWritingBase()) {
+ // If writing delta dirs, we need to make a clone of original options, to avoid polluting it for
+ // the base writer
+ if (options.isWritingBase()) {
+ if (options instanceof OrcOptions) {
+ writerOptions = ((OrcOptions) options).getOrcOptions();
+ }
+ if (writerOptions == null) {
+ writerOptions = OrcFile.writerOptions(options.getTableProperties(),
+ options.getConfiguration());
+ }
+ } else { // delta writer
+ AcidOutputFormat.Options optionsCloneForDelta = options.clone();
+
+ if (optionsCloneForDelta instanceof OrcOptions) {
+ writerOptions = ((OrcOptions) optionsCloneForDelta).getOrcOptions();
+ }
+ if (writerOptions == null) {
+ writerOptions = OrcFile.writerOptions(optionsCloneForDelta.getTableProperties(),
+ optionsCloneForDelta.getConfiguration());
+ }
+
+ // get buffer size and stripe size for base writer
+ int baseBufferSizeValue = writerOptions.getBufferSize();
+ long baseStripeSizeValue = writerOptions.getStripeSize();
+
+ // overwrite buffer size and stripe size for delta writer, based on BASE_DELTA_RATIO
+ int ratio = (int) OrcConf.BASE_DELTA_RATIO.getLong(options.getConfiguration());
+ writerOptions.bufferSize(baseBufferSizeValue / ratio);
+ writerOptions.stripeSize(baseStripeSizeValue / ratio);
writerOptions.blockPadding(false);
- writerOptions.bufferSize(DELTA_BUFFER_SIZE);
- writerOptions.stripeSize(DELTA_STRIPE_SIZE);
}
+ writerOptions.fileSystem(fs).callback(indexBuilder);
rowInspector = (StructObjectInspector)options.getInspector();
writerOptions.inspector(createEventSchema(findRecId(options.getInspector(),
options.getRecordIdColumn())));
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
index 0a61fb8..67c473e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
@@ -30,6 +30,7 @@ import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
@@ -197,6 +198,8 @@ public class TestOrcRecordUpdater {
}
Properties tblProps = new Properties();
tblProps.setProperty("orc.compress", "SNAPPY");
+ tblProps.setProperty("orc.compress.size", "8192");
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_ORC_BASE_DELTA_RATIO, 4);
AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
.filesystem(fs)
.bucket(10)
@@ -223,6 +226,7 @@ public class TestOrcRecordUpdater {
System.out.flush();
String outDump = new String(myOut.toByteArray());
assertEquals(true, outDump.contains("Compression: SNAPPY"));
+ assertEquals(true, outDump.contains("Compression size: 2048"));
System.setOut(origOut);
updater.close(false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out b/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
index 804e5e2..a460182 100644
--- a/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
+++ b/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
@@ -45,7 +45,7 @@ Table Parameters:
numFiles 3
numRows 0
rawDataSize 0
- totalSize 100540
+ totalSize 101147
transactional true
#### A masked pattern was here ####
@@ -75,9 +75,9 @@ Stage-0
File Output Operator [FS_3]
Limit [LIM_2] (rows=10 width=4)
Number of rows:10
- Select Operator [SEL_1] (rows=25135 width=4)
+ Select Operator [SEL_1] (rows=25286 width=4)
Output:["_col0"]
- TableScan [TS_0] (rows=25135 width=4)
+ TableScan [TS_0] (rows=25286 width=4)
default@acidtest1,acidtest1, ACID table,Tbl:COMPLETE,Col:NONE,Output:["c1"]
PREHOOK: query: select cast (c1 as string) from acidtest1 limit 10