You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2016/07/22 18:18:51 UTC

hive git commit: HIVE-14268 : INSERT-OVERWRITE is not generating an INSERT event during hive replication (Sushanth Sowmyan, reviewed by Alan Gates)

Repository: hive
Updated Branches:
  refs/heads/master a864bd39a -> b7f3d3a04


HIVE-14268 : INSERT-OVERWRITE is not generating an INSERT event during hive replication (Sushanth Sowmyan, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7f3d3a0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7f3d3a0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7f3d3a0

Branch: refs/heads/master
Commit: b7f3d3a04afd988bbf56c0f1a7e2e1fa8e89c119
Parents: a864bd3
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Fri Jul 22 11:18:04 2016 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Fri Jul 22 11:18:45 2016 -0700

----------------------------------------------------------------------
 .../listener/TestDbNotificationListener.java    | 29 ++++++++++++++++++--
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  4 ++-
 2 files changed, 30 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/b7f3d3a0/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 2d4c960..d0802d6 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -534,12 +534,15 @@ public class TestDbNotificationListener {
     driver.run("insert into table sip partition (ds) values (3, 'tomorrow')");
     driver.run("alter table sip drop partition (ds = 'tomorrow')");
 
+    driver.run("insert into table sip partition (ds) values (42, 'todaytwo')");
+    driver.run("insert overwrite table sip partition(ds='todaytwo') select c from sip where 'ds'='today'");
+
     NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
 
     for (NotificationEvent ne : rsp.getEvents()) LOG.debug("EVENT: " + ne.getMessage());
     // For reasons not clear to me there's one or more alter partitions after add partition and
     // insert.
-    assertEquals(19, rsp.getEventsSize());
+    assertEquals(24, rsp.getEventsSize());
     NotificationEvent event = rsp.getEvents().get(1);
     assertEquals(firstEventId + 2, event.getEventId());
     assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
@@ -569,7 +572,29 @@ public class TestDbNotificationListener {
     event = rsp.getEvents().get(18);
     assertEquals(firstEventId + 19, event.getEventId());
     assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT, event.getEventType());
-  }
+
+    event = rsp.getEvents().get(19);
+    assertEquals(firstEventId + 20, event.getEventId());
+    assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
+    event = rsp.getEvents().get(20);
+    assertEquals(firstEventId + 21, event.getEventId());
+    assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+
+    event = rsp.getEvents().get(21);
+    assertEquals(firstEventId + 22, event.getEventId());
+    assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+    assertTrue(event.getMessage().matches(".*\"files\":\\[\\].*")); // replace-overwrite introduces no new files
+    event = rsp.getEvents().get(22);
+    assertEquals(firstEventId + 23, event.getEventId());
+    assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+    event = rsp.getEvents().get(23);
+    assertEquals(firstEventId + 24, event.getEventId());
+    assertEquals(HCatConstants.HCAT_ALTER_PARTITION_EVENT, event.getEventType());
+    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
+
+   }
 
   @Test
   public void cleanupNotifs() throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/b7f3d3a0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 10f9cc7..9d927bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1554,8 +1554,10 @@ public class Hive {
       Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
       alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
       validatePartition(newTPart);
-      if (null != newFiles) {
+      if ((null != newFiles) || replace) {
         fireInsertEvent(tbl, partSpec, newFiles);
+      } else {
+        LOG.debug("No new files were created, and is not a replace. Skipping generating INSERT event.");
       }
 
       //column stats will be inaccurate