You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/01 00:03:07 UTC

[impala] 03/03: IMPALA-8455: Gracefully handle failed loads in GET_TABLES req

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit c1b0a073938c144e9bf33901bd4df6dcda0f09ec
Author: Bharath Vissapragada <bh...@cloudera.com>
AuthorDate: Mon Apr 29 21:30:05 2019 -0700

    IMPALA-8455: Gracefully handle failed loads in GET_TABLES req
    
    Some HS2 clients (like Hue) rely on the GET_TABLES request to
    populate table meta information. In local catalog mode, for any
    table loading failures, this call throws a cryptic exception
    that the callers do not expect.
    
    The original ImpaladCatalog implementation gracefully handled this
    by silently ignoring such tables. This patch replicates that
    behavior in LocalCatalog mode.
    
    Testing: Added a unit test that consistently fails without the patch.
    
    Change-Id: I89cc2c585aa83a9aea5343f0d0383ba72bafd618
    Reviewed-on: http://gerrit.cloudera.org:8080/13186
    Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../java/org/apache/impala/catalog/FeCatalog.java  |  3 +++
 .../apache/impala/catalog/local/LocalCatalog.java  | 10 +++++++++
 .../java/org/apache/impala/service/MetadataOp.java |  7 ++++--
 .../impala/catalog/local/LocalCatalogTest.java     | 25 +++++++++++++++++++++-
 4 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java b/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
index 10396f9..e0e63c2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
@@ -43,6 +43,9 @@ public interface FeCatalog {
   FeTable getTable(String db_name, String table_name)
       throws DatabaseNotFoundException;
 
+  /** @see Catalog#getTableNoThrow(String, String) */
+  FeTable getTableNoThrow(String db_name, String table_name);
+
   /** @see Catalog#getTCatalogObject(TCatalogObject) */
   TCatalogObject getTCatalogObject(TCatalogObject objectDesc)
       throws CatalogException;
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
index b2f3f9c..1868073 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
@@ -121,6 +121,16 @@ public class LocalCatalog implements FeCatalog {
   }
 
   @Override
+  public FeTable getTableNoThrow(String dbName, String tableName) {
+    try {
+      return getTable(dbName, tableName);
+    } catch (Exception e) {
+      // pass
+    }
+    return null;
+  }
+
+  @Override
   public TCatalogObject getTCatalogObject(TCatalogObject objectDesc)
       throws CatalogException {
     // TODO(todd): this probably makes the /catalog page not load with an error.
diff --git a/fe/src/main/java/org/apache/impala/service/MetadataOp.java b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
index fa4a0e7..3d85aa9 100644
--- a/fe/src/main/java/org/apache/impala/service/MetadataOp.java
+++ b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
@@ -301,8 +301,11 @@ public class MetadataOp {
         List<String> tableComments = Lists.newArrayList();
         List<String> tableTypes = Lists.newArrayList();
         for (String tabName: fe.getTableNames(db.getName(), tablePatternMatcher, user)) {
-          FeTable table = catalog.getTable(db.getName(), tabName);
-          if (table == null) continue;
+          FeTable table = catalog.getTableNoThrow(db.getName(), tabName);
+          if (table == null) {
+            result.missingTbls.add(new TableName(db.getName(), tabName));
+            continue;
+          }
 
           String comment = null;
           List<Column> columns = Lists.newArrayList();
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index a9f0d9d..990577b 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -22,8 +22,10 @@ import static org.junit.Assert.*;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.hive.service.rpc.thrift.TGetTablesReq;
 import org.apache.impala.analysis.Expr;
 import org.apache.impala.analysis.ToSqlUtils;
+import org.apache.impala.authorization.NoopAuthorizationFactory;
 import org.apache.impala.catalog.CatalogTest;
 import org.apache.impala.catalog.ColumnStats;
 import org.apache.impala.catalog.FeCatalogUtils;
@@ -36,7 +38,11 @@ import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
 import org.apache.impala.catalog.Type;
 import org.apache.impala.service.BackendConfig;
 import org.apache.impala.service.FeSupport;
+import org.apache.impala.service.Frontend;
 import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.thrift.TMetadataOpRequest;
+import org.apache.impala.thrift.TMetadataOpcode;
+import org.apache.impala.thrift.TResultRow;
 import org.apache.impala.thrift.TResultSet;
 import org.apache.impala.util.MetaStoreUtil;
 import org.apache.impala.util.PatternMatcher;
@@ -52,12 +58,14 @@ import com.google.common.collect.Iterables;
 public class LocalCatalogTest {
   private CatalogdMetaProvider provider_;
   private LocalCatalog catalog_;
+  private Frontend fe_;
 
   @Before
-  public void setupCatalog() {
+  public void setupCatalog() throws Exception {
     FeSupport.loadLibrary();
     provider_ = new CatalogdMetaProvider(BackendConfig.INSTANCE.getBackendCfg());
     catalog_ = new LocalCatalog(provider_, /*defaultKuduMasterHosts=*/null);
+    fe_ = new Frontend(new NoopAuthorizationFactory(), catalog_);
   }
 
   @Test
@@ -348,4 +356,19 @@ public class LocalCatalogTest {
     assertEquals(table_with_header.parseSkipHeaderLineCount(error), 1);
     assertEquals(error.length(), 0);
   }
+
+  /**
+   * Test GET_TABLES request on an Impala incompatible table. It should be silently
+   * ignored.
+   */
+  @Test
+  public void testGetTables() throws Exception {
+    TMetadataOpRequest req = new TMetadataOpRequest();
+    req.opcode = TMetadataOpcode.GET_TABLES;
+    req.get_tables_req = new TGetTablesReq();
+    req.get_tables_req.setSchemaName("functional");
+    req.get_tables_req.setTableName("bad_serde");
+    TResultSet resp = fe_.execHiveServer2MetadataOp(req);
+    assertEquals(0, resp.rows.size());
+  }
 }