You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/01 00:03:07 UTC
[impala] 03/03: IMPALA-8455: Gracefully handle failed loads in
GET_TABLES req
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c1b0a073938c144e9bf33901bd4df6dcda0f09ec
Author: Bharath Vissapragada <bh...@cloudera.com>
AuthorDate: Mon Apr 29 21:30:05 2019 -0700
IMPALA-8455: Gracefully handle failed loads in GET_TABLES req
Some HS2 clients (like Hue) rely on the GET_TABLES request to
populate table meta information. In local catalog mode, for any
table loading failures, this call throws a cryptic exception
that the callers do not expect.
The original ImpaladCatalog implementation gracefully handled this
by silently ignoring such tables. This patch replicates that
behavior in LocalCatalog mode.
Testing: Added a unit test that consistently fails without the patch.
Change-Id: I89cc2c585aa83a9aea5343f0d0383ba72bafd618
Reviewed-on: http://gerrit.cloudera.org:8080/13186
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/catalog/FeCatalog.java | 3 +++
.../apache/impala/catalog/local/LocalCatalog.java | 10 +++++++++
.../java/org/apache/impala/service/MetadataOp.java | 7 ++++--
.../impala/catalog/local/LocalCatalogTest.java | 25 +++++++++++++++++++++-
4 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java b/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
index 10396f9..e0e63c2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeCatalog.java
@@ -43,6 +43,9 @@ public interface FeCatalog {
FeTable getTable(String db_name, String table_name)
throws DatabaseNotFoundException;
+ /** @see Catalog#getTableNoThrow(String, String) */
+ FeTable getTableNoThrow(String db_name, String table_name);
+
/** @see Catalog#getTCatalogObject(TCatalogObject) */
TCatalogObject getTCatalogObject(TCatalogObject objectDesc)
throws CatalogException;
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
index b2f3f9c..1868073 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
@@ -121,6 +121,16 @@ public class LocalCatalog implements FeCatalog {
}
@Override
+ public FeTable getTableNoThrow(String dbName, String tableName) {
+ try {
+ return getTable(dbName, tableName);
+ } catch (Exception e) {
+ // pass
+ }
+ return null;
+ }
+
+ @Override
public TCatalogObject getTCatalogObject(TCatalogObject objectDesc)
throws CatalogException {
// TODO(todd): this probably makes the /catalog page not load with an error.
diff --git a/fe/src/main/java/org/apache/impala/service/MetadataOp.java b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
index fa4a0e7..3d85aa9 100644
--- a/fe/src/main/java/org/apache/impala/service/MetadataOp.java
+++ b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
@@ -301,8 +301,11 @@ public class MetadataOp {
List<String> tableComments = Lists.newArrayList();
List<String> tableTypes = Lists.newArrayList();
for (String tabName: fe.getTableNames(db.getName(), tablePatternMatcher, user)) {
- FeTable table = catalog.getTable(db.getName(), tabName);
- if (table == null) continue;
+ FeTable table = catalog.getTableNoThrow(db.getName(), tabName);
+ if (table == null) {
+ result.missingTbls.add(new TableName(db.getName(), tabName));
+ continue;
+ }
String comment = null;
List<Column> columns = Lists.newArrayList();
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index a9f0d9d..990577b 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -22,8 +22,10 @@ import static org.junit.Assert.*;
import java.util.List;
import java.util.Set;
+import org.apache.hive.service.rpc.thrift.TGetTablesReq;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.ToSqlUtils;
+import org.apache.impala.authorization.NoopAuthorizationFactory;
import org.apache.impala.catalog.CatalogTest;
import org.apache.impala.catalog.ColumnStats;
import org.apache.impala.catalog.FeCatalogUtils;
@@ -36,7 +38,11 @@ import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
import org.apache.impala.catalog.Type;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.service.FeSupport;
+import org.apache.impala.service.Frontend;
import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.thrift.TMetadataOpRequest;
+import org.apache.impala.thrift.TMetadataOpcode;
+import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TResultSet;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.util.PatternMatcher;
@@ -52,12 +58,14 @@ import com.google.common.collect.Iterables;
public class LocalCatalogTest {
private CatalogdMetaProvider provider_;
private LocalCatalog catalog_;
+ private Frontend fe_;
@Before
- public void setupCatalog() {
+ public void setupCatalog() throws Exception {
FeSupport.loadLibrary();
provider_ = new CatalogdMetaProvider(BackendConfig.INSTANCE.getBackendCfg());
catalog_ = new LocalCatalog(provider_, /*defaultKuduMasterHosts=*/null);
+ fe_ = new Frontend(new NoopAuthorizationFactory(), catalog_);
}
@Test
@@ -348,4 +356,19 @@ public class LocalCatalogTest {
assertEquals(table_with_header.parseSkipHeaderLineCount(error), 1);
assertEquals(error.length(), 0);
}
+
+ /**
+ * Test GET_TABLES request on an Impala incompatible table. It should be silently
+ * ignored.
+ */
+ @Test
+ public void testGetTables() throws Exception {
+ TMetadataOpRequest req = new TMetadataOpRequest();
+ req.opcode = TMetadataOpcode.GET_TABLES;
+ req.get_tables_req = new TGetTablesReq();
+ req.get_tables_req.setSchemaName("functional");
+ req.get_tables_req.setTableName("bad_serde");
+ TResultSet resp = fe_.execHiveServer2MetadataOp(req);
+ assertEquals(0, resp.rows.size());
+ }
}