You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/14 22:43:02 UTC

[impala] 02/06: IMPALA-7201. Support DDL with LocalCatalog enabled

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch 2.x
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 3afde5d99e7bc434358b813457d83cac4a6f086c
Author: Todd Lipcon <to...@cloudera.com>
AuthorDate: Fri Jun 22 16:33:44 2018 -0700

    IMPALA-7201. Support DDL with LocalCatalog enabled
    
    This fixes a couple issues with DDL commands when LocalCatalog is
    enabled:
    
    - updateCatalogCache() gets called after any DDL. Instead of throwing an
      exception, we can just no-op this by returning some fake result.
    
    - In order to support 'drop database' we need to properly implement the
      various function-related calls such that they don't throw exceptions.
      This changes them to be stubbed out as having no functions.
    
    - Fixes for 'alter view' and 'drop view' so that the underlying target
      table gets loaded by the catalogd before attempting the operation.
      Without this, in the LocalCatalog case, the catalogd would only have
      an IncompleteTable and these operations would fail with "unexpected
      table type" errors.
    
    With this patch I was able to run 'run-tests.py -k views' and 3/4
    passed. The one that failed depends on HBase tables, not yet
    implemented.
    
    Change-Id: Ic39c97a5f5ad145e03b96d1a470dc2dfa6ec71a5
    Reviewed-on: http://gerrit.cloudera.org:8080/10806
    Reviewed-by: Todd Lipcon <to...@apache.org>
    Tested-by: Todd Lipcon <to...@apache.org>
---
 .../org/apache/impala/catalog/local/LocalDb.java   | 19 +++++++++++++------
 .../apache/impala/service/CatalogOpExecutor.java   | 22 ++++++++++++++++++++--
 .../apache/impala/service/FeCatalogManager.java    |  7 +++++--
 3 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
index 2d223a6..6a779d7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
@@ -17,6 +17,7 @@
 
 package org.apache.impala.catalog.local;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -136,34 +137,40 @@ class LocalDb implements FeDb {
 
   @Override
   public Function getFunction(Function desc, CompareMode mode) {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return null;
   }
 
   @Override
   public List<Function> getFunctions(String functionName) {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return Collections.emptyList();
   }
 
   @Override
   public List<Function> getFunctions(
       TFunctionCategory category, String function) {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return Collections.emptyList();
   }
 
   @Override
   public List<Function> getFunctions(
       TFunctionCategory category, PatternMatcher patternMatcher) {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return Collections.emptyList();
   }
 
   @Override
   public int numFunctions() {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return 0;
   }
 
   @Override
   public boolean containsFunction(String function) {
-    throw new UnsupportedOperationException("TODO");
+    // TODO(todd): implement functions
+    return false;
   }
 
   @Override
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 88668d7..39cc108 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -696,8 +696,9 @@ public class CatalogOpExecutor {
     Preconditions.checkState(params.getColumns() != null &&
         params.getColumns().size() > 0,
           "Null or empty column list given as argument to DdlExecutor.alterView");
-    Table tbl = catalog_.getTable(tableName.getDb(), tableName.getTbl());
-    Preconditions.checkState(tbl instanceof View);
+    Table tbl = getExistingTable(tableName.getDb(), tableName.getTbl());
+    Preconditions.checkState(tbl instanceof View, "Expected view: %s",
+        tableName);
     tryLock(tbl);
     try {
       long newCatalogVersion = catalog_.incrementAndGetCatalogVersion();
@@ -1390,6 +1391,23 @@ public class CatalogOpExecutor {
     Preconditions.checkState(tableName != null && tableName.isFullyQualified());
     LOG.trace(String.format("Dropping table/view %s", tableName));
 
+    // If the table exists, ensure that it is loaded before we try to operate on it.
+    // We do this up here rather than down below to avoid doing too much table-loading
+    // work while holding the DDL lock. We can't simply use 'getExistingTable' because
+    // we rely on more granular checks to provide the correct summary message for
+    // the 'IF EXISTS' case.
+    //
+    // In the standard catalogd implementation, the table will most likely already
+    // be loaded because the planning phase on the impalad side triggered the loading.
+    // In the LocalCatalog configuration, however, this is often necessary.
+    try {
+      catalog_.getOrLoadTable(params.getTable_name().db_name,
+          params.getTable_name().table_name);
+    } catch (CatalogException e) {
+      // Ignore exceptions -- the above was just to trigger loading. Failure to load
+      // or non-existence of the database will be handled down below.
+    }
+
     TCatalogObject removedObject = new TCatalogObject();
     synchronized (metastoreDdlLock_) {
       Db db = catalog_.getDb(params.getTable_name().db_name);
diff --git a/fe/src/main/java/org/apache/impala/service/FeCatalogManager.java b/fe/src/main/java/org/apache/impala/service/FeCatalogManager.java
index 2bd44a4..ef8340d 100644
--- a/fe/src/main/java/org/apache/impala/service/FeCatalogManager.java
+++ b/fe/src/main/java/org/apache/impala/service/FeCatalogManager.java
@@ -22,6 +22,7 @@ import org.apache.impala.catalog.CatalogException;
 import org.apache.impala.catalog.FeCatalog;
 import org.apache.impala.catalog.ImpaladCatalog;
 import org.apache.impala.catalog.local.LocalCatalog;
+import org.apache.impala.thrift.TUniqueId;
 import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
 import org.apache.impala.thrift.TUpdateCatalogCacheResponse;
 import org.apache.thrift.TException;
@@ -127,8 +128,10 @@ public abstract class FeCatalogManager {
 
     @Override
     TUpdateCatalogCacheResponse updateCatalogCache(TUpdateCatalogCacheRequest req) {
-      throw new IllegalStateException(
-          "Unexpected call to updateCatalogCache() with local catalog enabled");
+      // TODO(todd) upon implementing caching, probably want to invalidate appropriate
+      // pieces of cached info. For now, this fake response seems enough to make
+      // the backend call site happy.
+      return new TUpdateCatalogCacheResponse(new TUniqueId(), -1, -1);
     }
   }