You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2022/05/18 08:00:01 UTC
[hive] branch master updated: HIVE-26231: Generate insert notification events when dynamic partition insert is done on existing partitions ( Sourabh Badhya, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a0cdd365e6c HIVE-26231: Generate insert notification events when dynamic partition insert is done on existing partitions ( Sourabh Badhya, reviewed by Denys Kuzmenko)
a0cdd365e6c is described below
commit a0cdd365e6cac90fde66483df834ca88578b30cf
Author: SourabhBadhya <42...@users.noreply.github.com>
AuthorDate: Wed May 18 13:29:49 2022 +0530
HIVE-26231: Generate insert notification events when dynamic partition insert is done on existing partitions ( Sourabh Badhya, reviewed by Denys Kuzmenko)
Closes #3294
---
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../insert_dynamic_partitions_notification_log.q | 15 +++++
...nsert_dynamic_partitions_notification_log.q.out | 70 ++++++++++++++++++++++
4 files changed, 87 insertions(+), 1 deletion(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 9c9e5db787d..de8c998c16b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -83,6 +83,7 @@ minillap.query.files=\
input36.q,\
input38.q,\
input5.q,\
+ insert_dynamic_partitions_notification_log.q,\
insert_into1.q,\
insert_into2.q,\
insert_into3.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 66b4cb2f1be..d2617ca0069 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3156,7 +3156,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (partitionDetails.newFiles != null) {
// If we already know the files from the direct insert manifest, use them
newFiles = partitionDetails.newFiles;
- } else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPartition == null) {
+ } else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
// Otherwise only collect them, if we are going to fire write notifications
newFiles = Collections.synchronizedList(new ArrayList<>());
}
diff --git a/ql/src/test/queries/clientpositive/insert_dynamic_partitions_notification_log.q b/ql/src/test/queries/clientpositive/insert_dynamic_partitions_notification_log.q
new file mode 100644
index 00000000000..0f9e737fb86
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_dynamic_partitions_notification_log.q
@@ -0,0 +1,15 @@
+--! qt:sysdb
+set hive.metastore.event.listeners=org.apache.hive.hcatalog.listener.DbNotificationListener;
+set hive.metastore.dml.events=true;
+
+CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC;
+
+INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);
+
+SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';
+
+INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);
+
+SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';
+
+DROP TABLE exttable;
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/insert_dynamic_partitions_notification_log.q.out b/ql/src/test/results/clientpositive/llap/insert_dynamic_partitions_notification_log.q.out
new file mode 100644
index 00000000000..223b8db8496
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/insert_dynamic_partitions_notification_log.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@exttable
+POSTHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@exttable
+PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@exttable
+POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@exttable
+POSTHOOK: Output: default@exttable@a=2
+POSTHOOK: Output: default@exttable@a=3
+POSTHOOK: Output: default@exttable@a=4
+POSTHOOK: Output: default@exttable@a=5
+POSTHOOK: Output: default@exttable@a=6
+POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
+PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+PREHOOK: type: QUERY
+PREHOOK: Input: sys@notification_log
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: sys@notification_log
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@exttable
+POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@exttable
+POSTHOOK: Output: default@exttable@a=2
+POSTHOOK: Output: default@exttable@a=3
+POSTHOOK: Output: default@exttable@a=4
+POSTHOOK: Output: default@exttable@a=5
+POSTHOOK: Output: default@exttable@a=6
+POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
+PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+PREHOOK: type: QUERY
+PREHOOK: Input: sys@notification_log
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: sys@notification_log
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+5
+PREHOOK: query: DROP TABLE exttable
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@exttable
+PREHOOK: Output: default@exttable
+POSTHOOK: query: DROP TABLE exttable
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@exttable
+POSTHOOK: Output: default@exttable