You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2019/06/03 10:55:27 UTC
[hive] branch master updated: HIVE-21784: Insert overwrite on an
acid (not mm) table is ineffective if the input is empty (Ivan Suller via
Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f62379b HIVE-21784: Insert overwrite on an acid (not mm) table is ineffective if the input is empty (Ivan Suller via Laszlo Bodor)
f62379b is described below
commit f62379ba279f41b843fcd5f3d4a107b6fcd04dec
Author: Ivan Suller <is...@cloudera.com>
AuthorDate: Mon Jun 3 12:49:19 2019 +0200
HIVE-21784: Insert overwrite on an acid (not mm) table is ineffective if the input is empty (Ivan Suller via Laszlo Bodor)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 12 +++++-
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 12 +-----
.../test/queries/clientpositive/insert_overwrite.q | 11 ++++++
.../clientpositive/llap/insert_overwrite.q.out | 43 ++++++++++++++++++++++
4 files changed, 65 insertions(+), 13 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 6472e80..2e4ef31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -573,10 +573,18 @@ public class OrcRecordUpdater implements RecordUpdater {
writer.close(); // normal close, when there are inserts.
}
} else {
- if (LOG.isDebugEnabled()) {
+ if (options.isWritingBase()) {
+ // With insert overwrite we need the empty file to delete the previous content of the table
+ LOG.debug("Empty file has been created for overwrite: {}", path);
+
+ OrcFile.WriterOptions wo = OrcFile.writerOptions(this.options.getConfiguration())
+ .inspector(rowInspector)
+ .callback(new OrcRecordUpdater.KeyIndexBuilder("testEmpty"));
+ OrcFile.createWriter(path, wo).close();
+ } else {
LOG.debug("No insert events in path: {}.. Deleting..", path);
+ fs.delete(path, false);
}
- fs.delete(path, false);
}
} else {
//so that we create empty bucket files when needed (but see HIVE-17138)
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index cbed1fc..8451462 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -538,7 +538,7 @@ public class TestOrcRawRecordMerger {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
OrcOutputFormat of = new OrcOutputFormat();
- Path root = new Path(tmpDir, "testEmpty").makeQualified(fs);
+ Path root = new Path(tmpDir, "testLogicalEmpty").makeQualified(fs);
fs.delete(root, true);
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
@@ -592,16 +592,6 @@ public class TestOrcRawRecordMerger {
.inspector(inspector).bucket(BUCKET).writingBase(true)
.maximumWriteId(100).finalDestination(root);
of.getRecordUpdater(root, options).close(false);
- {
- /*OrcRecordUpdater is inconsistent about when it creates empty files and when it does not.
- This creates an empty bucket. HIVE-17138*/
- OrcFile.WriterOptions wo = OrcFile.writerOptions(conf);
- wo.inspector(inspector);
- wo.callback(new OrcRecordUpdater.KeyIndexBuilder("testEmpty"));
- Writer w = OrcFile.createWriter(AcidUtils.createBucketFile(new Path(root,
- AcidUtils.baseDir(100)), BUCKET), wo);
- w.close();
- }
conf.set(ValidTxnList.VALID_TXNS_KEY,
new ValidReadTxnList(new long[0], new BitSet(), 1000, Long.MAX_VALUE).writeToString());
ValidWriteIdList writeIdList = new ValidReaderWriteIdList("testEmpty:200:" + Long.MAX_VALUE);
diff --git a/ql/src/test/queries/clientpositive/insert_overwrite.q b/ql/src/test/queries/clientpositive/insert_overwrite.q
index 6e4ead3..bc68217 100644
--- a/ql/src/test/queries/clientpositive/insert_overwrite.q
+++ b/ql/src/test/queries/clientpositive/insert_overwrite.q
@@ -20,6 +20,17 @@ SELECT count(*) FROM ext_non_part;
drop table ext_non_part;
+CREATE TABLE int_non_part (col string) STORED AS ORC TBLPROPERTIES ('transactional'='true');
+INSERT INTO int_non_part VALUES ('first'), ('second');
+
+INSERT OVERWRITE TABLE int_non_part SELECT * FROM b;
+
+-- should be 0
+SELECT count(*) FROM int_non_part;
+
+drop table int_non_part;
+
+
CREATE TABLE int_non_part (col string);
INSERT INTO int_non_part VALUES ('first'), ('second');
diff --git a/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out b/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out
index 22146cb..fbc3326 100644
--- a/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out
+++ b/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out
@@ -49,6 +49,49 @@ POSTHOOK: query: drop table ext_non_part
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@ext_non_part
POSTHOOK: Output: default@ext_non_part
+PREHOOK: query: CREATE TABLE int_non_part (col string) STORED AS ORC TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@int_non_part
+POSTHOOK: query: CREATE TABLE int_non_part (col string) STORED AS ORC TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@int_non_part
+PREHOOK: query: INSERT INTO int_non_part VALUES ('first'), ('second')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@int_non_part
+POSTHOOK: query: INSERT INTO int_non_part VALUES ('first'), ('second')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@int_non_part
+POSTHOOK: Lineage: int_non_part.col SCRIPT []
+PREHOOK: query: INSERT OVERWRITE TABLE int_non_part SELECT * FROM b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@b
+PREHOOK: Output: default@int_non_part
+POSTHOOK: query: INSERT OVERWRITE TABLE int_non_part SELECT * FROM b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@b
+POSTHOOK: Output: default@int_non_part
+POSTHOOK: Lineage: int_non_part.col SIMPLE [(b)b.FieldSchema(name:col, type:string, comment:null), ]
+PREHOOK: query: SELECT count(*) FROM int_non_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@int_non_part
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM int_non_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@int_non_part
+#### A masked pattern was here ####
+0
+PREHOOK: query: drop table int_non_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@int_non_part
+PREHOOK: Output: default@int_non_part
+POSTHOOK: query: drop table int_non_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@int_non_part
+POSTHOOK: Output: default@int_non_part
PREHOOK: query: CREATE TABLE int_non_part (col string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default