You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2016/07/22 18:18:51 UTC
hive git commit: HIVE-14268 : INSERT-OVERWRITE is not generating an
INSERT event during hive replication (Sushanth Sowmyan,
reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master a864bd39a -> b7f3d3a04
HIVE-14268 : INSERT-OVERWRITE is not generating an INSERT event during hive replication (Sushanth Sowmyan, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7f3d3a0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7f3d3a0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7f3d3a0
Branch: refs/heads/master
Commit: b7f3d3a04afd988bbf56c0f1a7e2e1fa8e89c119
Parents: a864bd3
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Fri Jul 22 11:18:04 2016 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Fri Jul 22 11:18:45 2016 -0700
----------------------------------------------------------------------
.../listener/TestDbNotificationListener.java | 29 ++++++++++++++++++--
.../apache/hadoop/hive/ql/metadata/Hive.java | 4 ++-
2 files changed, 30 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b7f3d3a0/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 2d4c960..d0802d6 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -534,12 +534,15 @@ public class TestDbNotificationListener {
driver.run("insert into table sip partition (ds) values (3, 'tomorrow')");
driver.run("alter table sip drop partition (ds = 'tomorrow')");
+ driver.run("insert into table sip partition (ds) values (42, 'todaytwo')");
+ driver.run("insert overwrite table sip partition(ds='todaytwo') select c from sip where 'ds'='today'");
+
NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
for (NotificationEvent ne : rsp.getEvents()) LOG.debug("EVENT: " + ne.getMessage());
// For reasons not clear to me there's one or more alter partitions after add partition and
// insert.
- assertEquals(19, rsp.getEventsSize());
+ assertEquals(24, rsp.getEventsSize());
NotificationEvent event = rsp.getEvents().get(1);
assertEquals(firstEventId + 2, event.getEventId());
assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
@@ -569,7 +572,29 @@ public class TestDbNotificationListener {
event = rsp.getEvents().get(18);
assertEquals(firstEventId + 19, event.getEventId());
assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT, event.getEventType());
- }
+
+ event = rsp.getEvents().get(19);
+ assertEquals(firstEventId + 20, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
+ event = rsp.getEvents().get(20);
+ assertEquals(firstEventId + 21, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+ assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+
+ event = rsp.getEvents().get(21);
+ assertEquals(firstEventId + 22, event.getEventId());
+ assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+ assertTrue(event.getMessage().matches(".*\"files\":\\[\\].*")); // replace-overwrite introduces no new files
+ event = rsp.getEvents().get(22);
+ assertEquals(firstEventId + 23, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+ assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+ event = rsp.getEvents().get(23);
+ assertEquals(firstEventId + 24, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+ assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+
+ }
@Test
public void cleanupNotifs() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/b7f3d3a0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 10f9cc7..9d927bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1554,8 +1554,10 @@ public class Hive {
Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
validatePartition(newTPart);
- if (null != newFiles) {
+ if ((null != newFiles) || replace) {
fireInsertEvent(tbl, partSpec, newFiles);
+ } else {
+ LOG.debug("No new files were created, and is not a replace. Skipping generating INSERT event.");
}
//column stats will be inaccurate