[impala] 02/04: IMPALA-10959: Reload MV as ACID tables

[ar...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

arawat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit dc8f375986321f2d7c1ed750e5c02c2e6f8cb4e8
Author: Yu-Wen Lai <yu...@cloudera.com>
AuthorDate: Tue Oct 5 15:42:47 2021 -0700

    IMPALA-10959: Reload MV as ACID tables
    
    We observed that the event processor is broken after receiving a
    partition event for materialized views (MV). This is because we are
    treating MV as view in Impala but Hive generates partition events for MV,
    which breaks current event processor.
    
    In this patch, we let partition events of MV follow the code path of ACID
    tables to reload the view. In the long term, we will need IMPALA-10723 to
    treat materialized view as a table.
    
    Tests:
    - manually testing
    
    Change-Id: Ibeab8cc53ad47d24df8baba81e1ec6ea4c80a084
    Reviewed-on: http://gerrit.cloudera.org:8080/17911
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Vihang Karajgaonkar <vi...@cloudera.com>
---
 .../impala/catalog/events/MetastoreEvents.java     | 36 ++++++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 93d3af0..59c381a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage;
 import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateDatabaseMessage;
 import org.apache.hadoop.hive.metastore.messaging.json.JSONDropDatabaseMessage;
 import org.apache.hadoop.hive.metastore.messaging.json.JSONDropTableMessage;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.impala.analysis.TableName;
 import org.apache.impala.catalog.CatalogException;
 import org.apache.impala.catalog.CatalogServiceCatalog;
@@ -960,8 +961,11 @@ public class MetastoreEvents {
         infoLog("Not processing the event as it is a self-event");
         return;
       }
-      // Reload the whole table if it's a transactional table.
-      if (AcidUtils.isTransactionalTable(msTbl_.getParameters())) {
+      // Reload the whole table if it's a transactional table or materialized view.
+      // Materialized views are treated as a special case because it causes problems
+      // on the reloading partition logic which expects it to be a HdfsTable.
+      if (AcidUtils.isTransactionalTable(msTbl_.getParameters())
+          || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
         insertPartition_ = null;
       }
 
@@ -1534,8 +1538,13 @@ public class MetastoreEvents {
         return;
       }
       try {
-        // Reload the whole table if it's a transactional table.
-        if (AcidUtils.isTransactionalTable(msTbl_.getParameters()) && !isSelfEvent()) {
+        // Reload the whole table if it's a transactional table or materialized view.
+        // Materialized views are treated as a special case because it's possible to
+        // receive partition event on MVs, but they are regular views in Impala. That
+        // cause problems on the reloading partition logic which expects it to be a
+        // HdfsTable.
+        if ((AcidUtils.isTransactionalTable(msTbl_.getParameters()) && !isSelfEvent())
+            || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
           reloadTableFromCatalog("ADD_PARTITION", true);
         } else {
           // HMS adds partitions in a transactional way. This means there may be multiple
@@ -1669,8 +1678,12 @@ public class MetastoreEvents {
         return;
       }
 
-      // Reload the whole table if it's a transactional table.
-      if (AcidUtils.isTransactionalTable(msTbl_.getParameters())) {
+      // Reload the whole table if it's a transactional table or materialized view.
+      // Materialized views are treated as a special case because it's possible to receive
+      // partition event on MVs, but they are regular views in Impala. That cause problems
+      // on the reloading partition logic which expects it to be a HdfsTable.
+      if (AcidUtils.isTransactionalTable(msTbl_.getParameters())
+          || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
         reloadTableFromCatalog("ALTER_PARTITION", true);
       } else {
         // Refresh the partition that was altered.
@@ -1886,10 +1899,13 @@ public class MetastoreEvents {
         infoLog("Partition list is empty. Ignoring this event.");
       }
       try {
-        // Reload the whole table if it's a transactional table. In case of transactional
-        // tables we rely on the self-event evaluation since there is no fine-grained
-        // partition level refresh.
-        if (AcidUtils.isTransactionalTable(msTbl_.getParameters())) {
+        // Reload the whole table if it's a transactional table or materialized view.
+        // Materialized views are treated as a special case because it's possible to
+        // receive partition event on MVs, but they are regular views in Impala. That
+        // cause problems on the reloading partition logic which expects it to be a
+        // HdfsTable.
+        if (AcidUtils.isTransactionalTable(msTbl_.getParameters())
+            || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
           reloadTableFromCatalog("DROP_PARTITION", true);
         } else {
           int numPartsRemoved = catalogOpExecutor_