You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by to...@apache.org on 2018/07/13 17:42:21 UTC
[2/3] impala git commit: IMPALA-7140 (part 8): support views in
LocalCatalog
IMPALA-7140 (part 8): support views in LocalCatalog
This adds basic support for loading views in LocalCatalog. Tested with a
small unit test and also verified from the shell that I can select from
a view.
Change-Id: Ib3516b9ceff6dce12ded68d93afde09728627e08
Reviewed-on: http://gerrit.cloudera.org:8080/10805
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f14ae3fc
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f14ae3fc
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f14ae3fc
Branch: refs/heads/master
Commit: f14ae3fc3a16bf707f5c23ec60816c4df0996682
Parents: 7f8a4d3
Author: Todd Lipcon <to...@cloudera.com>
Authored: Fri Jun 22 10:53:23 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Jul 13 17:39:06 2018 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/catalog/View.java | 54 ++++++-------
.../apache/impala/catalog/local/LocalTable.java | 3 +-
.../apache/impala/catalog/local/LocalView.java | 83 ++++++++++++++++++++
.../impala/catalog/local/LocalCatalogTest.java | 9 +++
4 files changed, 116 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/View.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/View.java b/fe/src/main/java/org/apache/impala/catalog/View.java
index 66d942a..4026d6a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/View.java
+++ b/fe/src/main/java/org/apache/impala/catalog/View.java
@@ -45,24 +45,6 @@ import com.google.common.collect.Lists;
*/
public class View extends Table implements FeView {
- // The original SQL-string given as view definition. Set during analysis.
- // Corresponds to Hive's viewOriginalText.
- private String originalViewDef_;
-
- // Query statement (as SQL string) that defines the View for view substitution.
- // It is a transformation of the original view definition, e.g., to enforce the
- // explicit column definitions even if the original view definition has explicit
- // column aliases.
- // If column definitions were given, then this "expanded" view definition
- // wraps the original view definition in a select stmt as follows.
- //
- // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ...
- // FROM (originalViewDef) AS viewName
- //
- // Corresponds to Hive's viewExpandedText, but is not identical to the SQL
- // Hive would produce in view creation.
- private String inlineViewDef_;
-
// View definition created by parsing inlineViewDef_ into a QueryStmt.
private QueryStmt queryStmt_;
@@ -117,7 +99,7 @@ public class View extends Table implements FeView {
numClusteringCols_ = 0;
tableStats_ = new TTableStats(-1);
tableStats_.setTotal_file_bytes(-1);
- init();
+ queryStmt_ = parseViewDef(this);
} catch (TableLoadingException e) {
throw e;
} catch (Exception e) {
@@ -128,22 +110,32 @@ public class View extends Table implements FeView {
@Override
protected void loadFromThrift(TTable t) throws TableLoadingException {
super.loadFromThrift(t);
- init();
+ queryStmt_ = parseViewDef(this);
}
/**
- * Initializes the originalViewDef_, inlineViewDef_, and queryStmt_ members
- * by parsing the expanded view definition SQL-string.
+ * Parse the expanded view definition SQL-string.
* Throws a TableLoadingException if there was any error parsing the
* the SQL or if the view definition did not parse into a QueryStmt.
*/
- private void init() throws TableLoadingException {
- // Set view-definition SQL strings.
- originalViewDef_ = getMetaStoreTable().getViewOriginalText();
- inlineViewDef_ = getMetaStoreTable().getViewExpandedText();
+ public static QueryStmt parseViewDef(FeView view) throws TableLoadingException {
+ // Query statement (as SQL string) that defines the View for view substitution.
+ // It is a transformation of the original view definition, e.g., to enforce the
+ // explicit column definitions even if the original view definition has explicit
+ // column aliases.
+ // If column definitions were given, then this "expanded" view definition
+ // wraps the original view definition in a select stmt as follows.
+ //
+ // SELECT viewName.origCol1 AS colDesc1, viewName.origCol2 AS colDesc2, ...
+ // FROM (originalViewDef) AS viewName
+ //
+ // Corresponds to Hive's viewExpandedText, but is not identical to the SQL
+ // Hive would produce in view creation.
+ String inlineViewDef = view.getMetaStoreTable().getViewExpandedText();
+
// Parse the expanded view definition SQL-string into a QueryStmt and
// populate a view definition.
- SqlScanner input = new SqlScanner(new StringReader(inlineViewDef_));
+ SqlScanner input = new SqlScanner(new StringReader(inlineViewDef));
SqlParser parser = new SqlParser(input);
ParseNode node = null;
try {
@@ -153,14 +145,14 @@ public class View extends Table implements FeView {
// of tables that the user triggering this load may not have privileges on.
throw new TableLoadingException(
String.format("Failed to parse view-definition statement of view: " +
- "%s.%s", db_.getName(), name_));
+ "%s", view.getFullName()));
}
// Make sure the view definition parses to a query statement.
if (!(node instanceof QueryStmt)) {
- throw new TableLoadingException(String.format("View definition of %s.%s " +
- "is not a query statement", db_.getName(), name_));
+ throw new TableLoadingException(String.format("View definition of %s " +
+ "is not a query statement", view.getFullName()));
}
- queryStmt_ = (QueryStmt) node;
+ return (QueryStmt) node;
}
@Override
http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
index 4b57cb8..d128ca2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
@@ -40,7 +40,6 @@ import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.TableLoadingException;
-import org.apache.impala.catalog.View;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TTableStats;
import org.apache.log4j.Logger;
@@ -73,7 +72,7 @@ abstract class LocalTable implements FeTable {
LocalTable t = null;
Table msTbl = schemaInfo.msTable_;
if (TableType.valueOf(msTbl.getTableType()) == TableType.VIRTUAL_VIEW) {
- // TODO(todd) support View
+ t = new LocalView(db, tblName, schemaInfo);
} else if (HBaseTable.isHBaseTable(msTbl)) {
// TODO(todd) support HBase table
} else if (KuduTable.isKuduTable(msTbl)) {
http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java
new file mode 100644
index 0000000..c380eb1
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalView.java
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog.local;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.impala.analysis.QueryStmt;
+import org.apache.impala.catalog.FeView;
+import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.View;
+import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.thrift.TTableDescriptor;
+
+/**
+ * View implementation corresponding to the LocalCatalog.
+ *
+ * NOTE: this does _not_ implement "local views" in the sense of a view defined
+ * in the scope of a query using a WITH expression.
+ */
+public class LocalView extends LocalTable implements FeView {
+ private final QueryStmt queryStmt_;
+
+ public LocalView(LocalDb db, String tblName, SchemaInfo schemaInfo) {
+ super(db, tblName, schemaInfo);
+
+ try {
+ queryStmt_ = View.parseViewDef(this);
+ } catch (TableLoadingException e) {
+ throw new LocalCatalogException(e);
+ }
+ }
+
+ @Override
+ public TTableDescriptor toThriftDescriptor(int tableId,
+ Set<Long> referencedPartitions) {
+ // Views are always resolved into underlying tables before being serialized.
+ throw new IllegalStateException("Cannot call toThriftDescriptor() on a view.");
+ }
+
+ // NOTE: "local view" in this context means "a view defined local to a query"
+ // whereas "local" in the class name "LocalView" means "view from the
+ // LocalCatalog catalog implementation".
+ @Override
+ public boolean isLocalView() {
+ // TODO: the org.apache.impala.catalog.View class is still used for local views
+ // even in the LocalCatalog implementation.
+ return false;
+ }
+
+ @Override
+ public QueryStmt getQueryStmt() {
+ return queryStmt_;
+ }
+
+ @Override
+ public List<String> getColLabels() {
+ // Explicit column labels are only used by local views.
+ // Returning null indicates that the column labels are derived
+ // from the underlying statement.
+ return null;
+ }
+
+ @Override
+ public TCatalogObjectType getCatalogObjectType() {
+ return TCatalogObjectType.VIEW;
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/f14ae3fc/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index e334d20..d85b8e3 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -29,9 +29,11 @@ import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeFsPartition;
import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.FeTable;
+import org.apache.impala.catalog.FeView;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
import org.apache.impala.catalog.Type;
import org.apache.impala.service.FeSupport;
+import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TResultSet;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.util.PatternMatcher;
@@ -167,4 +169,11 @@ public class LocalCatalogTest {
assertEquals(10210, stats.getNumDistinctValues());
assertEquals(-1, stats.getNumNulls());
}
+
+ @Test
+ public void testView() throws Exception {
+ FeView v = (FeView) catalog_.getTable("functional", "alltypes_view");
+ assertEquals(TCatalogObjectType.VIEW, v.getCatalogObjectType());
+ assertEquals("SELECT * FROM functional.alltypes", v.getQueryStmt().toSql());
+ }
}