You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/18 18:52:37 UTC
[41/48] hive git commit: HIVE-20165: Enable ZLIB for streaming ingest
(Prasanth Jayachandran, via Gopal V)
HIVE-20165: Enable ZLIB for streaming ingest (Prasanth Jayachandran, via Gopal V)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bac1d98c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bac1d98c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bac1d98c
Branch: refs/heads/master-txnstats
Commit: bac1d98c5e91cdb39567f21b2068e7951a93ef44
Parents: 4fcf3d7
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Jul 17 20:17:55 2018 -0700
Committer: Gopal V <go...@apache.org>
Committed: Tue Jul 17 20:17:55 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 1 -
.../apache/hive/streaming/TestStreaming.java | 21 ++++++++++----------
2 files changed, 10 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bac1d98c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 5590470..98f5df1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -342,7 +342,6 @@ public class OrcRecordUpdater implements RecordUpdater {
writerOptions.blockPadding(false);
if (optionsCloneForDelta.getConfiguration().getBoolean(
HiveConf.ConfVars.HIVE_ORC_DELTA_STREAMING_OPTIMIZATIONS_ENABLED.varname, false)) {
- writerOptions.compress(CompressionKind.NONE);
writerOptions.encodingStrategy(org.apache.orc.OrcFile.EncodingStrategy.SPEED);
writerOptions.rowIndexStride(0);
writerOptions.getConfiguration().set(OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getAttribute(), "-1.0");
http://git-wip-us.apache.org/repos/asf/hive/blob/bac1d98c/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
index 1f05d88..8b5e508 100644
--- a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
+++ b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
@@ -2072,20 +2072,19 @@ public class TestStreaming {
System.setOut(origOut);
String outDump = new String(myOut.toByteArray());
- // make sure delta files are written with no indexes, no compression and no dictionary
- // no compression
- Assert.assertEquals(true, outDump.contains("Compression: NONE"));
+ // make sure delta files are written with no indexes and no dictionary
+ Assert.assertEquals(true, outDump.contains("Compression: ZLIB"));
// no stats/indexes
Assert.assertEquals(true, outDump.contains("Column 0: count: 0 hasNull: false"));
- Assert.assertEquals(true, outDump.contains("Column 1: count: 0 hasNull: false bytesOnDisk: 12 sum: 0"));
- Assert.assertEquals(true, outDump.contains("Column 2: count: 0 hasNull: false bytesOnDisk: 12 sum: 0"));
- Assert.assertEquals(true, outDump.contains("Column 3: count: 0 hasNull: false bytesOnDisk: 24 sum: 0"));
- Assert.assertEquals(true, outDump.contains("Column 4: count: 0 hasNull: false bytesOnDisk: 14 sum: 0"));
- Assert.assertEquals(true, outDump.contains("Column 5: count: 0 hasNull: false bytesOnDisk: 12 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 1: count: 0 hasNull: false bytesOnDisk: 15 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 2: count: 0 hasNull: false bytesOnDisk: 15 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 3: count: 0 hasNull: false bytesOnDisk: 19 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 4: count: 0 hasNull: false bytesOnDisk: 17 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 5: count: 0 hasNull: false bytesOnDisk: 15 sum: 0"));
Assert.assertEquals(true, outDump.contains("Column 6: count: 0 hasNull: false"));
- Assert.assertEquals(true, outDump.contains("Column 7: count: 0 hasNull: false bytesOnDisk: 11864"));
- Assert.assertEquals(true, outDump.contains("Column 8: count: 0 hasNull: false bytesOnDisk: 2033 sum: 0"));
- Assert.assertEquals(true, outDump.contains("Column 9: count: 0 hasNull: false bytesOnDisk: 13629"));
+ Assert.assertEquals(true, outDump.contains("Column 7: count: 0 hasNull: false bytesOnDisk: 3929"));
+ Assert.assertEquals(true, outDump.contains("Column 8: count: 0 hasNull: false bytesOnDisk: 1484 sum: 0"));
+ Assert.assertEquals(true, outDump.contains("Column 9: count: 0 hasNull: false bytesOnDisk: 816"));
// no dictionary
Assert.assertEquals(true, outDump.contains("Encoding column 7: DIRECT_V2"));
Assert.assertEquals(true, outDump.contains("Encoding column 9: DIRECT_V2"));