[impala] 02/04: IMPALA-8369 : Fix for tests failing with incompatible column changes

[to...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

todd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 4fdd9aeaacf5b9a80ed9f011eda3490fd830ac89
Author: Vihang Karajgaonkar <vi...@cloudera.com>
AuthorDate: Mon May 6 17:42:59 2019 -0700

    IMPALA-8369 : Fix for tests failing with incompatible column changes
    
    In Hive-3 the configuration for allowing users to make incompatible
    column type changes was disabled by default. In Hive-2 this was allowed.
    Some of the tests like data_errors/test_data_errors.py and
    metadata/test_compute_stats.py make changes to column types which are
    disallowed by HMS-3 by default. This change adds a configuration option
    in hive-site.xml to allow making incompatible changes to column types so
    that we can run the existing tests with HMS-3.
    
    Also, in HMS-3 there are certain new event types (OPEN_TXN, COMMIT_TXN,
    etc) which may not have dbname set. This breaks the assumption in the
    code in EventProcessor which expects dbName_ to be not null at all
    times. This patch also makes changes in the EventProcessor so that such
    Ignored events do not fail precondition checks during event processing.
    
    Change-Id: I488121f21d9b35d33dd003b2670bc0bbe1fee4b6
    Reviewed-on: http://gerrit.cloudera.org:8080/13254
    Reviewed-by: Todd Lipcon <to...@apache.org>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../main/java/org/apache/impala/catalog/events/MetastoreEvents.java | 5 ++++-
 fe/src/test/resources/hive-site.xml.py                              | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 513390a..7a6457d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -289,7 +289,8 @@ public class MetastoreEvents {
       this.event_ = event;
       this.eventId_ = event_.getEventId();
       this.eventType_ = MetastoreEventType.from(event.getEventType());
-      this.dbName_ = Preconditions.checkNotNull(event.getDbName());
+      // certain event types in Hive-3 like COMMIT_TXN may not have dbName set
+      this.dbName_ = event.getDbName();
       this.tblName_ = event.getTableName();
       this.metastoreNotificationEvent_ = event;
       this.metrics_ = metrics;
@@ -483,6 +484,7 @@ public class MetastoreEvents {
     private MetastoreTableEvent(CatalogServiceCatalog catalogServiceCatalog,
         Metrics metrics, NotificationEvent event) {
       super(catalogServiceCatalog, metrics, event);
+      Preconditions.checkNotNull(dbName_, debugString("Database name cannot be null"));
       tblName_ = Preconditions.checkNotNull(event.getTableName());
       debugLog("Creating event {} of type {} on table {}", eventId_, eventType_,
           getFullyQualifiedTblName());
@@ -607,6 +609,7 @@ public class MetastoreEvents {
     MetastoreDatabaseEvent(CatalogServiceCatalog catalogServiceCatalog, Metrics metrics,
         NotificationEvent event) {
       super(catalogServiceCatalog, metrics, event);
+      Preconditions.checkNotNull(dbName_, debugString("Database name cannot be null"));
       debugLog("Creating event {} of type {} on database {}", eventId_,
               eventType_, dbName_);
     }
diff --git a/fe/src/test/resources/hive-site.xml.py b/fe/src/test/resources/hive-site.xml.py
index 66e62a7..65d65e4 100644
--- a/fe/src/test/resources/hive-site.xml.py
+++ b/fe/src/test/resources/hive-site.xml.py
@@ -84,10 +84,14 @@ if hive_major_version >= 3:
    # We run YARN with Tez on the classpath directly
    'tez.ignore.lib.uris': 'true',
    'tez.use.cluster.hadoop-libs': 'true',
+   # Some of the tests change the columns in a incompatible manner
+   # (eg. string to timestamp) this is disallowed by default in Hive-3 which causes
+   # these tests to fail. We disable this behavior in minicluster to keep running the
+   # same tests on both hms-2 and hms-3
+   'hive.metastore.disallow.incompatible.col.type.changes': 'false'
   })
 else:
   CONFIG.update({
-   # TODO(vihang) Disabled for HMS3.
    'hive.metastore.event.listeners': 'org.apache.sentry.binding.metastore.SentrySyncHMSNotificationsPostEventListener',
   })