You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2021/10/08 16:18:58 UTC
[hive] branch master updated: HIVE-25590: Able to create views
referencing temporary tables and materialized views (Krisztian Kasa,
reviewed by Zoltan Haindrich, Alessandro Solimando)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ac40207 HIVE-25590: Able to create views referencing temporary tables and materialized views (Krisztian Kasa, reviewed by Zoltan Haindrich, Alessandro Solimando)
ac40207 is described below
commit ac4020789149115c71bcee9b85577fff7dd01f46
Author: Krisztian Kasa <ka...@gmail.com>
AuthorDate: Fri Oct 8 18:18:34 2021 +0200
HIVE-25590: Able to create views referencing temporary tables and materialized views (Krisztian Kasa, reviewed by Zoltan Haindrich, Alessandro Solimando)
---
.../view/create/AbstractCreateViewAnalyzer.java | 21 +++++++------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 24 ++++++++--------------
.../queries/clientnegative/create_view_failure11.q | 5 +++++
.../queries/clientnegative/create_view_failure12.q | 7 +++++++
.../materialized_view_create_failure.q | 12 +++++++++++
.../clientnegative/create_view_failure10.q.out | 2 +-
...failure10.q.out => create_view_failure11.q.out} | 2 +-
...failure10.q.out => create_view_failure12.q.out} | 10 ++++++++-
.../materialized_view_create_failure.q.out | 17 +++++++++++++++
...aterialized_view_no_transactional_rewrite.q.out | 2 +-
10 files changed, 68 insertions(+), 34 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java
index 1f9f103..bd0f6bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.ddl.view.create;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -42,7 +41,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils;
/**
* Abstract ancestor of analyzers that can create a view.
*/
-abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer {
+public abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer {
AbstractCreateViewAnalyzer(QueryState queryState) throws SemanticException {
super(queryState);
}
@@ -85,26 +84,20 @@ abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer {
}
}
- protected void validateTablesUsed(SemanticAnalyzer analyzer) throws SemanticException {
+ public static void validateTablesUsed(SemanticAnalyzer analyzer) throws SemanticException {
// Do not allow view to be defined on temp table or other materialized view
- Set<String> tableAliases = analyzer.getQB().getTabAliases();
- for (String alias : tableAliases) {
- if (SemanticAnalyzer.DUMMY_TABLE.equals(alias)) {
+ for (TableScanOperator ts : analyzer.getTopOps().values()) {
+ Table table = ts.getConf().getTableMetadata();
+ if (SemanticAnalyzer.DUMMY_TABLE.equals(table.getTableName())) {
continue;
}
- Table table = null;
- try {
- table = analyzer.getTableObjectByName(analyzer.getQB().getTabNameForAlias(alias));
- } catch (HiveException ex) {
- throw new SemanticException(ex);
- }
if (table.isTemporary()) {
- throw new SemanticException("View definition references temporary table " + alias);
+ throw new SemanticException("View definition references temporary table " + table.getCompleteName());
}
if (table.isMaterializedView()) {
- throw new SemanticException("View definition references materialized view " + alias);
+ throw new SemanticException("View definition references materialized view " + table.getCompleteName());
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b210b94..b2dc3c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -23,12 +23,12 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.DYNAMICPARTITIONCONV
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
+import static org.apache.hadoop.hive.ql.ddl.view.create.AbstractCreateViewAnalyzer.validateTablesUsed;
import static org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.NON_FK_FILTERED;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.AccessControlException;
-import java.time.ZoneId;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
@@ -13943,25 +13943,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throws SemanticException {
try {
// Do not allow view to be defined on temp table or other materialized view
- Set<String> tableAliases = qb.getTabAliases();
- for (String alias : tableAliases) {
- try {
- if (DUMMY_TABLE.equals(alias)) {
+ validateTablesUsed(this);
+ if (createVwDesc.isRewriteEnabled()) {
+ for (TableScanOperator ts : topOps.values()) {
+ Table table = ts.getConf().getTableMetadata();
+ if (SemanticAnalyzer.DUMMY_TABLE.equals(table.getTableName())) {
continue;
}
- Table table = getTableObjectByName(qb.getTabNameForAlias(alias));
- if (table.isTemporary()) {
- throw new SemanticException("View definition references temporary table " + alias);
- }
- if (table.isMaterializedView()) {
- throw new SemanticException("View definition references materialized view " + alias);
- }
- if (createVwDesc.isRewriteEnabled() && !AcidUtils.isTransactionalTable(table)) {
+ if (!AcidUtils.isTransactionalTable(table)) {
throw new SemanticException("Automatic rewriting for materialized view cannot "
- + "be enabled if the materialized view uses non-transactional tables");
+ + "be enabled if the materialized view uses non-transactional tables");
}
- } catch (HiveException ex) {
- throw new SemanticException(ex);
}
}
diff --git a/ql/src/test/queries/clientnegative/create_view_failure11.q b/ql/src/test/queries/clientnegative/create_view_failure11.q
new file mode 100644
index 0000000..c164e3f
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/create_view_failure11.q
@@ -0,0 +1,5 @@
+-- CREATE VIEW should fail if it references a temp table in a subquery
+create temporary table tmp1 (c1 string, c2 string);
+
+create view tmp1_view as
+select subq.c1 from (select c1, c2 from tmp1) subq;
diff --git a/ql/src/test/queries/clientnegative/create_view_failure12.q b/ql/src/test/queries/clientnegative/create_view_failure12.q
new file mode 100644
index 0000000..32339d7
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/create_view_failure12.q
@@ -0,0 +1,7 @@
+-- CREATE VIEW should fail if it joins temp tables
+create temporary table tmp1 (c1 string, c2 string);
+create temporary table tmp2 (c1 string, c2 string);
+
+create view tmp1_view as
+select tmp1.c1 from tmp1
+join tmp2 on tmp1.c1 = tmp2.c1;
diff --git a/ql/src/test/queries/clientnegative/materialized_view_create_failure.q b/ql/src/test/queries/clientnegative/materialized_view_create_failure.q
new file mode 100644
index 0000000..c0eeea9
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/materialized_view_create_failure.q
@@ -0,0 +1,12 @@
+-- CREATE MATERIALIZED VIEW should fail if it references a temp table
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true');
+create table t2 (a int) stored as orc tblproperties ('transactional'='true');
+
+create materialized view mv1 as
+select t2.a from t1
+join t2 on (t1.a = t2.a);
+
+drop materialized view mv1;
diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure10.q.out
index 5aedfd3..7ba0844 100644
--- a/ql/src/test/results/clientnegative/create_view_failure10.q.out
+++ b/ql/src/test/results/clientnegative/create_view_failure10.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tmp1
-FAILED: SemanticException View definition references temporary table tmp1
+FAILED: SemanticException View definition references temporary table default@tmp1
diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure11.q.out
similarity index 95%
copy from ql/src/test/results/clientnegative/create_view_failure10.q.out
copy to ql/src/test/results/clientnegative/create_view_failure11.q.out
index 5aedfd3..7ba0844 100644
--- a/ql/src/test/results/clientnegative/create_view_failure10.q.out
+++ b/ql/src/test/results/clientnegative/create_view_failure11.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tmp1
-FAILED: SemanticException View definition references temporary table tmp1
+FAILED: SemanticException View definition references temporary table default@tmp1
diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure12.q.out
similarity index 53%
copy from ql/src/test/results/clientnegative/create_view_failure10.q.out
copy to ql/src/test/results/clientnegative/create_view_failure12.q.out
index 5aedfd3..651f794 100644
--- a/ql/src/test/results/clientnegative/create_view_failure10.q.out
+++ b/ql/src/test/results/clientnegative/create_view_failure12.q.out
@@ -6,4 +6,12 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tmp1
-FAILED: SemanticException View definition references temporary table tmp1
+PREHOOK: query: create temporary table tmp2 (c1 string, c2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp2
+POSTHOOK: query: create temporary table tmp2 (c1 string, c2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp2
+FAILED: SemanticException View definition references temporary table default@tmp1
diff --git a/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out b/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out
new file mode 100644
index 0000000..e420202
--- /dev/null
+++ b/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out
@@ -0,0 +1,17 @@
+PREHOOK: query: create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: create table t2 (a int) stored as orc tblproperties ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2
+POSTHOOK: query: create table t2 (a int) stored as orc tblproperties ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2
+FAILED: SemanticException View definition references temporary table default@t1
diff --git a/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out
index abb3d96..6384c96 100644
--- a/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out
+++ b/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out
@@ -17,4 +17,4 @@ POSTHOOK: Output: default@cmv_basetable
POSTHOOK: Lineage: cmv_basetable.a SCRIPT []
POSTHOOK: Lineage: cmv_basetable.b SCRIPT []
POSTHOOK: Lineage: cmv_basetable.c SCRIPT []
-FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Automatic rewriting for materialized view cannot be enabled if the materialized view uses non-transactional tables
+FAILED: SemanticException Automatic rewriting for materialized view cannot be enabled if the materialized view uses non-transactional tables