You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/11/21 21:44:47 UTC
[5/5] hive git commit: HIVE-14487: Add REBUILD statement for
materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-14487: Add REBUILD statement for materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a9e9d600
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a9e9d600
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a9e9d600
Branch: refs/heads/master
Commit: a9e9d60000f542cd2244825752b0a5fa6546770f
Parents: 8fcc7f3
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Nov 16 12:59:56 2017 -0800
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Nov 21 13:44:26 2017 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 9 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 108 +-
.../repl/bootstrap/load/LoadConstraint.java | 18 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 35 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 10 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 45 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 8 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 35 +-
.../hive/ql/parse/SemanticAnalyzerFactory.java | 5 +
.../repl/dump/io/ConstraintsSerializer.java | 2 +-
.../hadoop/hive/ql/plan/BasicStatsWork.java | 9 +-
.../hadoop/hive/ql/plan/CreateViewDesc.java | 115 +-
.../org/apache/hadoop/hive/ql/plan/DDLWork.java | 8 -
.../hadoop/hive/ql/plan/ImportTableDesc.java | 2 +-
.../hadoop/hive/ql/plan/LoadFileDesc.java | 20 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 5 +
...alized_view_authorization_rebuild_no_grant.q | 20 +
...erialized_view_authorization_rebuild_other.q | 14 +
.../materialized_view_create_rewrite.q | 2 +
.../materialized_view_create_rewrite_3.q | 78 +
.../clientnegative/create_view_failure1.q.out | 2 +-
...ed_view_authorization_rebuild_no_grant.q.out | 40 +
...lized_view_authorization_rebuild_other.q.out | 28 +
.../materialized_view_create_rewrite.q.out | 6 +-
.../results/clientpositive/create_view.q.out | 2 -
.../clientpositive/create_view_translate.q.out | 2 -
.../results/clientpositive/explain_ddl.q.out | 2 -
.../llap/materialized_view_create.q.out | 207 +++
.../llap/materialized_view_create_rewrite.q.out | 501 ++++++
.../materialized_view_create_rewrite_2.q.out | 714 ++++++++
.../materialized_view_create_rewrite_3.q.out | 504 ++++++
...erialized_view_create_rewrite_multi_db.q.out | 157 ++
.../llap/materialized_view_describe.q.out | 354 ++++
.../llap/materialized_view_drop.q.out | 39 +
.../llap/materialized_view_rewrite_ssb.q.out | 1671 +++++++++++++++++
.../llap/materialized_view_rewrite_ssb_2.q.out | 1677 ++++++++++++++++++
.../llap/selectDistinctStar.q.out | 4 -
.../clientpositive/llap/union_top_level.q.out | 2 -
.../clientpositive/llap/vector_windowing.q.out | 4 -
.../materialized_view_create.q.out | 1 +
.../materialized_view_create_rewrite.q.out | 6 +-
.../materialized_view_create_rewrite_3.q.out | 531 ++++++
...erialized_view_create_rewrite_multi_db.q.out | 2 +
.../materialized_view_describe.q.out | 5 +
.../clientpositive/spark/union_top_level.q.out | 2 -
45 files changed, 6839 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3b1005f..5d17733 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -206,6 +206,15 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
load_dyn_part2.q,\
load_dyn_part3.q,\
lvj_mapjoin.q,\
+ materialized_view_create_rewrite.q,\
+ materialized_view_create_rewrite_3.q,\
+ materialized_view_describe.q,\
+ materialized_view_rewrite_ssb.q,\
+ materialized_view_create.q,\
+ materialized_view_create_rewrite_2.q,\
+ materialized_view_create_rewrite_multi_db.q,\
+ materialized_view_drop.q,\
+ materialized_view_rewrite_ssb_2.q,\
mapjoin2.q,\
mapjoin3.q,\
mapjoin_decimal.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index b3d7a03..17640f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -142,7 +142,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker;
-import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
@@ -205,7 +204,6 @@ import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.MsckDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
-import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.PrincipalDesc;
import org.apache.hadoop.hive.ql.plan.PrivilegeDesc;
import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
@@ -5043,8 +5041,28 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
*/
private int createView(Hive db, CreateViewDesc crtView) throws HiveException {
Table oldview = db.getTable(crtView.getViewName(), false);
- if (crtView.getOrReplace() && oldview != null) {
- if (!crtView.isMaterialized()) {
+ if (oldview != null) {
+ // Check whether we are replicating
+ if (crtView.getReplicationSpec().isInReplicationScope()) {
+ // if this is a replication spec, then replace-mode semantics might apply.
+ if (crtView.getReplicationSpec().allowEventReplacementInto(oldview.getParameters())){
+ crtView.setReplace(true); // we replace existing view.
+ } else {
+ LOG.debug("DDLTask: Create View is skipped as view {} is newer than update",
+ crtView.getViewName()); // no replacement, the existing table state is newer than our update.
+ return 0;
+ }
+ }
+
+ if (!crtView.isReplace()) {
+ // View already exists, thus we should be replacing
+ throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(crtView.getViewName()));
+ }
+
+ if (crtView.isMaterialized()) {
+ // This is a replace/rebuild, so we need an exclusive lock
+ addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE));
+ } else {
// replace existing view
// remove the existing partition columns from the field schema
oldview.setViewOriginalText(crtView.getViewOriginalText());
@@ -5070,88 +5088,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
throw new HiveException(e);
}
addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK));
- } else {
- // This is a replace, so we need an exclusive lock
- addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE));
}
} else {
- // create new view
- Table tbl = db.newTable(crtView.getViewName());
- tbl.setViewOriginalText(crtView.getViewOriginalText());
- tbl.setViewExpandedText(crtView.getViewExpandedText());
- if (crtView.isMaterialized()) {
- tbl.setRewriteEnabled(crtView.isRewriteEnabled());
- tbl.setTableType(TableType.MATERIALIZED_VIEW);
- } else {
- tbl.setTableType(TableType.VIRTUAL_VIEW);
- }
- tbl.setSerializationLib(null);
- tbl.clearSerDeInfo();
- tbl.setFields(crtView.getSchema());
- if (crtView.getComment() != null) {
- tbl.setProperty("comment", crtView.getComment());
- }
- if (crtView.getTblProps() != null) {
- tbl.getTTable().getParameters().putAll(crtView.getTblProps());
- }
-
- if (crtView.getPartCols() != null) {
- tbl.setPartCols(crtView.getPartCols());
- }
-
- if (crtView.getInputFormat() != null) {
- tbl.setInputFormatClass(crtView.getInputFormat());
- }
-
- if (crtView.getOutputFormat() != null) {
- tbl.setOutputFormatClass(crtView.getOutputFormat());
- }
-
- if (crtView.isMaterialized()) {
- if (crtView.getLocation() != null) {
- tbl.setDataLocation(new Path(crtView.getLocation()));
- }
-
- if (crtView.getStorageHandler() != null) {
- tbl.setProperty(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE,
- crtView.getStorageHandler());
- }
- HiveStorageHandler storageHandler = tbl.getStorageHandler();
-
- /*
- * If the user didn't specify a SerDe, we use the default.
- */
- String serDeClassName;
- if (crtView.getSerde() == null) {
- if (storageHandler == null) {
- serDeClassName = PlanUtils.getDefaultSerDe().getName();
- LOG.info("Default to {} for materialized view {}", serDeClassName,
- crtView.getViewName());
- } else {
- serDeClassName = storageHandler.getSerDeClass().getName();
- LOG.info("Use StorageHandler-supplied {} for materialized view {}",
- serDeClassName, crtView.getViewName());
- }
- } else {
- // let's validate that the serde exists
- serDeClassName = crtView.getSerde();
- DDLTask.validateSerDe(serDeClassName, conf);
- }
- tbl.setSerializationLib(serDeClassName);
-
- // To remain consistent, we need to set input and output formats both
- // at the table level and the storage handler level.
- tbl.setInputFormatClass(crtView.getInputFormat());
- tbl.setOutputFormatClass(crtView.getOutputFormat());
- if (crtView.getInputFormat() != null && !crtView.getInputFormat().isEmpty()) {
- tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName());
- }
- if (crtView.getOutputFormat() != null && !crtView.getOutputFormat().isEmpty()) {
- tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName());
- }
- }
-
+ // We create new view
+ Table tbl = crtView.toTable(conf);
db.createTable(tbl, crtView.getIfNotExists());
// Add to cache if it is a materialized view
if (tbl.isMaterializedView()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java
index fc2aa8d..200ac0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java
@@ -82,15 +82,6 @@ public class LoadConstraint {
dbNameToLoadIn, null, fromPath.toString(), null, pkDumpMetaData, context.hiveConf,
context.hiveDb, null, LOG)));
- AddForeignKeyHandler fkHandler = new AddForeignKeyHandler();
- DumpMetaData fkDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_FOREIGNKEY, Long.MAX_VALUE, Long.MAX_VALUE, null,
- context.hiveConf);
- fkDumpMetaData.setPayload(fksString);
- tasks.addAll(fkHandler.handle(
- new MessageHandler.Context(
- dbNameToLoadIn, null, fromPath.toString(), null, fkDumpMetaData, context.hiveConf,
- context.hiveDb, null, LOG)));
-
AddUniqueConstraintHandler ukHandler = new AddUniqueConstraintHandler();
DumpMetaData ukDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_UNIQUECONSTRAINT, Long.MAX_VALUE, Long.MAX_VALUE, null,
context.hiveConf);
@@ -109,6 +100,15 @@ public class LoadConstraint {
dbNameToLoadIn, null, fromPath.toString(), null, nnDumpMetaData, context.hiveConf,
context.hiveDb, null, LOG)));
+ AddForeignKeyHandler fkHandler = new AddForeignKeyHandler();
+ DumpMetaData fkDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_FOREIGNKEY, Long.MAX_VALUE, Long.MAX_VALUE, null,
+ context.hiveConf);
+ fkDumpMetaData.setPayload(fksString);
+ tasks.addAll(fkHandler.handle(
+ new MessageHandler.Context(
+ dbNameToLoadIn, null, fromPath.toString(), null, fkDumpMetaData, context.hiveConf,
+ context.hiveDb, null, LOG)));
+
tasks.forEach(tracker::addTask);
return tracker;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 67739a1..be1d4b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1500,14 +1500,33 @@ public final class GenMapRedUtils {
statsWork = new BasicStatsWork(mvWork.getLoadFileWork());
truncate = true;
- if (mvWork.getLoadFileWork().getCtasCreateTableDesc() == null) {
- throw new RuntimeException("unexpected; this should be a CTAS - however no desc present");
- }
- try {
- table = mvWork.getLoadFileWork().getCtasCreateTableDesc().toTable(hconf);
- } catch (HiveException e) {
- LOG.debug("can't pre-create table", e);
- table = null;
+ if (mvWork.getLoadFileWork().getCtasCreateTableDesc() != null) {
+ try {
+ table = mvWork.getLoadFileWork().getCtasCreateTableDesc().toTable(hconf);
+ } catch (HiveException e) {
+ LOG.debug("can't pre-create table for CTAS", e);
+ table = null;
+ }
+ } else if (mvWork.getLoadFileWork().getCreateViewDesc() != null) {
+ if (mvWork.getLoadFileWork().getCreateViewDesc().isReplace()) {
+ // ALTER MV ... REBUILD
+ String tableName = mvWork.getLoadFileWork().getCreateViewDesc().getViewName();
+ try {
+ table = Hive.get().getTable(tableName);
+ } catch (HiveException e) {
+ throw new RuntimeException("unexpected; MV should be present already..: " + tableName, e);
+ }
+ } else {
+ // CREATE MATERIALIZED VIEW ...
+ try {
+ table = mvWork.getLoadFileWork().getCreateViewDesc().toTable(hconf);
+ } catch (HiveException e) {
+ LOG.debug("can't pre-create table for MV", e);
+ table = null;
+ }
+ }
+ } else {
+ throw new RuntimeException("unexpected; this should be a CTAS or a CREATE/REBUILD MV - however no desc present");
}
}
assert statsWork != null : "Error when generating StatsTask";
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index e9ae590..d36d24d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -1164,7 +1164,9 @@ public abstract class BaseSemanticAnalyzer {
|| ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION
|| ast.getToken().getType() == HiveParser.TOK_TABTYPE
|| ast.getToken().getType() == HiveParser.TOK_CREATETABLE
- || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW);
+ || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW
+ || (ast.getToken().getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW &&
+ ast.getChild(1).getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD));
int childIndex = 0;
numDynParts = 0;
@@ -1177,7 +1179,8 @@ public abstract class BaseSemanticAnalyzer {
+ tableName;
}
if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE &&
- ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW) {
+ ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW &&
+ ast.getToken().getType() != HiveParser.TOK_ALTER_MATERIALIZED_VIEW) {
tableHandle = db.getTable(tableName);
}
} catch (InvalidTableException ite) {
@@ -1190,7 +1193,8 @@ public abstract class BaseSemanticAnalyzer {
// get partition metadata if partition specified
if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE &&
- ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW) {
+ ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW &&
+ ast.getToken().getType() != HiveParser.TOK_ALTER_MATERIALIZED_VIEW) {
childIndex = 1;
ASTNode partspec = (ASTNode) ast.getChild(1);
partitions = new ArrayList<Partition>();
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 55b1da9..76c82e2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -370,6 +370,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
disableJoinMerge = true;
boolean reAnalyzeAST = false;
final boolean materializedView = getQB().isMaterializedView();
+ final boolean rebuild = materializedView && createVwDesc.isReplace();
try {
if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
@@ -389,7 +390,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
ASTNode newAST = getOptimizedAST();
// 1.1. Fix up the query for insert/ctas/materialized views
- newAST = fixUpAfterCbo(ast, newAST, cboCtx);
+ if (!rebuild) {
+ // If it is not a MATERIALIZED VIEW...REBUILD
+ newAST = fixUpAfterCbo(ast, newAST, cboCtx);
+ }
// 2. Regen OP plan from optimized AST
if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) {
@@ -399,25 +403,32 @@ public class CalcitePlanner extends SemanticAnalyzer {
throw new CalciteViewSemanticException(e.getMessage());
}
} else if (cboCtx.type == PreCboCtx.Type.VIEW && materializedView) {
- // Store text of the ORIGINAL QUERY
- String originalText = ctx.getTokenRewriteStream().toString(
- cboCtx.nodeOfInterest.getTokenStartIndex(),
- cboCtx.nodeOfInterest.getTokenStopIndex());
- unparseTranslator.applyTranslations(ctx.getTokenRewriteStream());
- String expandedText = ctx.getTokenRewriteStream().toString(
- cboCtx.nodeOfInterest.getTokenStartIndex(),
- cboCtx.nodeOfInterest.getTokenStopIndex());
- // Redo create-table/view analysis, because it's not part of
- // doPhase1.
- // Use the REWRITTEN AST
- init(false);
- setAST(newAST);
- newAST = reAnalyzeViewAfterCbo(newAST);
+ if (rebuild) {
+ // Use the CREATE MATERIALIZED VIEW...REBUILD
+ init(false);
+ setAST(ast);
+ reAnalyzeViewAfterCbo(ast);
+ } else {
+ // Store text of the ORIGINAL QUERY
+ String originalText = ctx.getTokenRewriteStream().toString(
+ cboCtx.nodeOfInterest.getTokenStartIndex(),
+ cboCtx.nodeOfInterest.getTokenStopIndex());
+ unparseTranslator.applyTranslations(ctx.getTokenRewriteStream());
+ String expandedText = ctx.getTokenRewriteStream().toString(
+ cboCtx.nodeOfInterest.getTokenStartIndex(),
+ cboCtx.nodeOfInterest.getTokenStopIndex());
+ // Redo create-table/view analysis, because it's not part of
+ // doPhase1.
+ // Use the REWRITTEN AST
+ init(false);
+ setAST(newAST);
+ newAST = reAnalyzeViewAfterCbo(newAST);
+ createVwDesc.setViewOriginalText(originalText);
+ createVwDesc.setViewExpandedText(expandedText);
+ }
viewSelect = newAST;
viewsExpanded = new ArrayList<>();
viewsExpanded.add(createVwDesc.getViewName());
- createVwDesc.setViewOriginalText(originalText);
- createVwDesc.setViewExpandedText(expandedText);
} else if (cboCtx.type == PreCboCtx.Type.CTAS) {
// CTAS
init(false);
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index f3f1a7e..f90bb8a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -262,6 +262,7 @@ TOK_CREATE_MATERIALIZED_VIEW;
TOK_DROP_MATERIALIZED_VIEW;
TOK_ALTER_MATERIALIZED_VIEW;
TOK_ALTER_MATERIALIZED_VIEW_REWRITE;
+TOK_ALTER_MATERIALIZED_VIEW_REBUILD;
TOK_REWRITE_ENABLED;
TOK_REWRITE_DISABLED;
TOK_VIEWPARTCOLS;
@@ -1366,6 +1367,7 @@ alterMaterializedViewStatementSuffix
@init { pushMsg("alter materialized view statement", state); }
@after { popMsg(state); }
: alterMaterializedViewSuffixRewrite
+ | alterMaterializedViewSuffixRebuild
;
alterIndexStatementSuffix
@@ -1542,6 +1544,12 @@ alterMaterializedViewSuffixRewrite
-> ^(TOK_ALTER_MATERIALIZED_VIEW_REWRITE $mvRewriteFlag)
;
+alterMaterializedViewSuffixRebuild
+@init { pushMsg("alter materialized view rebuild statement", state); }
+@after { popMsg(state); }
+ : KW_REBUILD -> ^(TOK_ALTER_MATERIALIZED_VIEW_REBUILD)
+ ;
+
alterStatementSuffixSerdeProperties
@init { pushMsg("alter serdes statement", state); }
@after { popMsg(state); }
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b66817f..b323ede 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -11365,6 +11365,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// 3. analyze create view command
if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW ||
ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW ||
+ (ast.getToken().getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW &&
+ ast.getChild(1).getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD) ||
(ast.getToken().getType() == HiveParser.TOK_ALTERVIEW &&
ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
child = analyzeCreateView(ast, qb, plannerCtx);
@@ -11750,6 +11752,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
protected void saveViewDefinition() throws SemanticException {
+ if (createVwDesc.isMaterialized() && createVwDesc.isReplace()) {
+ // This is a rebuild, there's nothing to do here.
+ return;
+ }
+
// Make a copy of the statement's result schema, since we may
// modify it below as part of imposing view column names.
List<FieldSchema> derivedSchema =
@@ -12594,6 +12601,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, String> tblProps = null;
List<String> partColNames = null;
boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW;
+ boolean isRebuild = false;
String location = null;
RowFormatParams rowFormatParams = new RowFormatParams();
StorageFormat storageFormat = new StorageFormat(conf);
@@ -12616,6 +12624,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
case HiveParser.TOK_ORREPLACE:
orReplace = true;
break;
+ case HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD:
+ isMaterialized = true;
+ isRebuild = true;
+ break;
case HiveParser.TOK_QUERY:
// For CBO
if (plannerCtx != null) {
@@ -12673,7 +12685,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (isMaterialized) {
createVwDesc = new CreateViewDesc(
dbDotTable, cols, comment, tblProps, partColNames,
- ifNotExists, orReplace, rewriteEnabled, isAlterViewAs,
+ ifNotExists, isRebuild, rewriteEnabled, isAlterViewAs,
storageFormat.getInputFormat(), storageFormat.getOutputFormat(),
location, storageFormat.getSerde(), storageFormat.getStorageHandler(),
storageFormat.getSerdeProps());
@@ -12691,6 +12703,25 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
qb.setViewDesc(createVwDesc);
+ if (isRebuild) {
+ // We need to go lookup the table and get the select statement and then parse it.
+ try {
+ Table tab = getTableObjectByName(dbDotTable, true);
+ String viewText = tab.getViewOriginalText();
+ if (viewText.trim().isEmpty()) {
+ throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY);
+ }
+ Context ctx = new Context(queryState.getConf());
+ selectStmt = ParseUtils.parse(viewText, ctx);
+ // For CBO
+ if (plannerCtx != null) {
+ plannerCtx.setViewToken(selectStmt);
+ }
+ } catch (Exception e) {
+ throw new SemanticException(e);
+ }
+ }
+
return selectStmt;
}
@@ -12727,7 +12758,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
//replace view
- if (createVwDesc.getOrReplace() && oldView != null) {
+ if (createVwDesc.isReplace() && oldView != null) {
// Don't allow swapping between virtual and materialized view in replace
if (oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && createVwDesc.isMaterialized()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
index d139155..a3b3287 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
@@ -273,6 +273,11 @@ public final class SemanticAnalyzerFactory {
opType = commandType.get(child.getType());
queryState.setCommandType(opType);
return new DDLSemanticAnalyzer(queryState);
+ case HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD:
+ opType = commandType.get(child.getType());
+ queryState.setCommandType(opType);
+ return HiveConf.getBoolVar(queryState.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED) ?
+ new CalcitePlanner(queryState) : new SemanticAnalyzer(queryState);
}
// Operation not recognized, set to null and let upper level handle this case
queryState.setCommandType(null);
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java
index 2ae9f58..267b406 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java
@@ -64,8 +64,8 @@ public class ConstraintsSerializer implements JsonWriter.Serializer {
nnsString = MessageFactory.getInstance().buildAddNotNullConstraintMessage(nns).toString();
}
writer.jsonGenerator.writeStringField("pks", pksString);
- writer.jsonGenerator.writeStringField("fks", fksString);
writer.jsonGenerator.writeStringField("uks", uksString);
writer.jsonGenerator.writeStringField("nns", nnsString);
+ writer.jsonGenerator.writeStringField("fks", fksString);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
index 0621bd4..26bb3e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java
@@ -179,6 +179,11 @@ public class BasicStatsWork implements Serializable {
if (getLoadFileDesc() != null && getLoadFileDesc().getCtasCreateTableDesc() != null) {
return true;
}
+ // ALTER MV ... REBUILD
+ if (getLoadFileDesc() != null && getLoadFileDesc().getCreateViewDesc() != null &&
+ getLoadFileDesc().getCreateViewDesc().isReplace()) {
+ return true;
+ }
return false;
}
@@ -188,8 +193,10 @@ public class BasicStatsWork implements Serializable {
return work.getLoadTableDesc().getTable().getTableName();
} else if (work.getTableSpecs() != null) {
return work.getTableSpecs().tableName;
- } else {
+ } else if (getLoadFileDesc().getCtasCreateTableDesc() != null) {
return getLoadFileDesc().getCtasCreateTableDesc().getTableName();
+ } else {
+ return getLoadFileDesc().getCreateViewDesc().getViewName();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
index 9425f6e..09aa82f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
@@ -22,10 +22,20 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -35,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level;
@Explain(displayName = "Create View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateViewDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
+ private static Logger LOG = LoggerFactory.getLogger(CreateViewDesc.class);
private String viewName;
private String originalText;
@@ -46,7 +57,7 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
private List<FieldSchema> partCols;
private String comment;
private boolean ifNotExists;
- private boolean orReplace;
+ private boolean replace;
private boolean isAlterViewAs;
private boolean isMaterialized;
private String inputFormat;
@@ -83,7 +94,7 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
*/
public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment,
Map<String, String> tblProps, List<String> partColNames,
- boolean ifNotExists, boolean orReplace, boolean rewriteEnabled, boolean isAlterViewAs,
+ boolean ifNotExists, boolean replace, boolean rewriteEnabled, boolean isAlterViewAs,
String inputFormat, String outputFormat, String location,
String serde, String storageHandler, Map<String, String> serdeProps) {
this.viewName = viewName;
@@ -92,7 +103,7 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
this.partColNames = partColNames;
this.comment = comment;
this.ifNotExists = ifNotExists;
- this.orReplace = orReplace;
+ this.replace = replace;
this.isMaterialized = true;
this.rewriteEnabled = rewriteEnabled;
this.isAlterViewAs = isAlterViewAs;
@@ -128,7 +139,7 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
this.partColNames = partColNames;
this.comment = comment;
this.ifNotExists = ifNotExists;
- this.orReplace = orReplace;
+ this.replace = orReplace;
this.isAlterViewAs = isAlterViewAs;
this.isMaterialized = false;
this.rewriteEnabled = false;
@@ -164,7 +175,7 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
this.expandedText = expandedText;
}
- @Explain(displayName = "rewrite enabled")
+ @Explain(displayName = "rewrite enabled", displayOnlyOnTrue = true)
public boolean isRewriteEnabled() {
return rewriteEnabled;
}
@@ -234,13 +245,13 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
this.ifNotExists = ifNotExists;
}
- @Explain(displayName = "or replace")
- public boolean getOrReplace() {
- return orReplace;
+ @Explain(displayName = "replace", displayOnlyOnTrue = true)
+ public boolean isReplace() {
+ return replace;
}
- public void setOrReplace(boolean orReplace) {
- this.orReplace = orReplace;
+ public void setReplace(boolean replace) {
+ this.replace = replace;
}
@Explain(displayName = "is alter view as select", displayOnlyOnTrue = true)
@@ -308,4 +319,88 @@ public class CreateViewDesc extends DDLDesc implements Serializable {
}
return this.replicationSpec;
}
+
+ public Table toTable(HiveConf conf) throws HiveException {
+ String[] names = Utilities.getDbTableName(getViewName());
+ String databaseName = names[0];
+ String tableName = names[1];
+
+ Table tbl = new Table(databaseName, tableName);
+ tbl.setViewOriginalText(getViewOriginalText());
+ tbl.setViewExpandedText(getViewExpandedText());
+ if (isMaterialized()) {
+ tbl.setRewriteEnabled(isRewriteEnabled());
+ tbl.setTableType(TableType.MATERIALIZED_VIEW);
+ } else {
+ tbl.setTableType(TableType.VIRTUAL_VIEW);
+ }
+ tbl.setSerializationLib(null);
+ tbl.clearSerDeInfo();
+ tbl.setFields(getSchema());
+ if (getComment() != null) {
+ tbl.setProperty("comment", getComment());
+ }
+ if (getTblProps() != null) {
+ tbl.getTTable().getParameters().putAll(getTblProps());
+ }
+
+ if (getPartCols() != null) {
+ tbl.setPartCols(getPartCols());
+ }
+
+ if (getInputFormat() != null) {
+ tbl.setInputFormatClass(getInputFormat());
+ }
+
+ if (getOutputFormat() != null) {
+ tbl.setOutputFormatClass(getOutputFormat());
+ }
+
+ if (isMaterialized()) {
+ if (getLocation() != null) {
+ tbl.setDataLocation(new Path(getLocation()));
+ }
+
+ if (getStorageHandler() != null) {
+ tbl.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE,
+ getStorageHandler());
+ }
+ HiveStorageHandler storageHandler = tbl.getStorageHandler();
+
+ /*
+ * If the user didn't specify a SerDe, we use the default.
+ */
+ String serDeClassName;
+ if (getSerde() == null) {
+ if (storageHandler == null) {
+ serDeClassName = PlanUtils.getDefaultSerDe().getName();
+ LOG.info("Default to {} for materialized view {}", serDeClassName,
+ getViewName());
+ } else {
+ serDeClassName = storageHandler.getSerDeClass().getName();
+ LOG.info("Use StorageHandler-supplied {} for materialized view {}",
+ serDeClassName, getViewName());
+ }
+ } else {
+ // let's validate that the serde exists
+ serDeClassName = getSerde();
+ DDLTask.validateSerDe(serDeClassName, conf);
+ }
+ tbl.setSerializationLib(serDeClassName);
+
+ // To remain consistent, we need to set input and output formats both
+ // at the table level and the storage handler level.
+ tbl.setInputFormatClass(getInputFormat());
+ tbl.setOutputFormatClass(getOutputFormat());
+ if (getInputFormat() != null && !getInputFormat().isEmpty()) {
+ tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName());
+ }
+ if (getOutputFormat() != null && !getOutputFormat().isEmpty()) {
+ tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName());
+ }
+ }
+
+ return tbl;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
index dc8f172..a9b39be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
@@ -1114,14 +1114,6 @@ public class DDLWork implements Serializable {
this.showTblPropertiesDesc = showTblPropertiesDesc;
}
- public CreateViewDesc getCreateVwDesc() {
- return createVwDesc;
- }
-
- public void setCreateVwDesc(CreateViewDesc createVwDesc) {
- this.createVwDesc = createVwDesc;
- }
-
public void setDescFunctionDesc(DescFunctionDesc descFunctionDesc) {
this.descFunctionDesc = descFunctionDesc;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
index 1770046..33e30bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
@@ -303,7 +303,7 @@ public class ImportTableDesc {
createTblDesc.setReplaceMode(replaceMode);
break;
case VIEW:
- createViewDesc.setOrReplace(replaceMode);
+ createViewDesc.setReplace(replaceMode);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
index c09589c..c3dab35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
@@ -36,6 +36,7 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
private String columns;
private String columnTypes;
private transient CreateTableDesc ctasCreateTableDesc;
+ private transient CreateViewDesc createViewDesc;
private boolean isMmCtas;
public LoadFileDesc(final LoadFileDesc o) {
@@ -47,21 +48,26 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
this.columnTypes = o.columnTypes;
this.isMmCtas = o.isMmCtas;
this.ctasCreateTableDesc = o.ctasCreateTableDesc;
+ this.createViewDesc = o.createViewDesc;
}
- public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc,
- final Path sourcePath, final Path targetDir, final boolean isDfsDir,
+ public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc,
+ final Path sourcePath, final Path targetDir, final boolean isDfsDir,
final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas);
- if (createTableDesc != null && createTableDesc.isCTAS()) {
- ctasCreateTableDesc = createTableDesc;
+ if (createTableDesc != null && createTableDesc.isCTAS()) {
+ this.ctasCreateTableDesc = createTableDesc;
+ }
+ if (createViewDesc != null && createViewDesc.isMaterialized()) {
+ this.createViewDesc = createViewDesc;
}
}
public LoadFileDesc(final Path sourcePath, final Path targetDir,
- final boolean isDfsDir, final String columns, final String columnTypes, boolean isMmCtas) {
+ final boolean isDfsDir, final String columns, final String columnTypes, boolean isMmCtas) {
this(sourcePath, targetDir, isDfsDir, columns, columnTypes, AcidUtils.Operation.NOT_ACID, isMmCtas);
}
+
private LoadFileDesc(final Path sourcePath, final Path targetDir,
final boolean isDfsDir, final String columns,
final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
@@ -128,6 +134,10 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
return ctasCreateTableDesc;
}
+ public CreateViewDesc getCreateViewDesc() {
+ return createViewDesc;
+ }
+
public boolean isMmCtas() {
return isMmCtas;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 3c1e92a..81cc279 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -442,6 +442,11 @@ public final class PlanUtils {
crtViewDesc.getStorageHandler());
}
+ if (crtViewDesc.getViewName() != null && crtViewDesc.isMaterialized()) {
+ properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,
+ crtViewDesc.getViewName());
+ }
+
if (crtViewDesc.getTblProps() != null) {
properties.putAll(crtViewDesc.getTblProps());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q
new file mode 100644
index 0000000..a2e7d38
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q
@@ -0,0 +1,20 @@
+set hive.test.authz.sstd.hs2.mode=true;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
+set hive.security.authorization.enabled=true;
+set user.name=user1;
+
+create table amvrng_table (a int, b varchar(256), c decimal(10,2));
+
+insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8);
+
+grant select on table amvrng_table to user user2 with grant option;
+
+set user.name=user2;
+create materialized view amvrng_mat_view as select a, c from amvrng_table;
+
+set user.name=user1;
+revoke grant option for select on table amvrng_table from user user2;
+
+set user.name=user2;
+alter materialized view amvrng_mat_view rebuild;
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q
new file mode 100644
index 0000000..7c2d145
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q
@@ -0,0 +1,14 @@
+set hive.test.authz.sstd.hs2.mode=true;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
+set hive.security.authorization.enabled=true;
+set user.name=user1;
+
+create table amvro_table (a int, b varchar(256), c decimal(10,2));
+
+insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8);
+
+create materialized view amvro_mat_view as select a, c from amvro_table;
+
+set user.name=user2;
+alter materialized view amvro_mat_view rebuild;
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q
index 1749cb0..761903f 100644
--- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q
+++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q
@@ -1,3 +1,5 @@
+-- SORT_QUERY_RESULTS
+
set hive.strict.checks.cartesian.product=false;
set hive.materializedview.rewriting=true;
set hive.stats.column.autogather=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q
new file mode 100644
index 0000000..6462d9a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q
@@ -0,0 +1,78 @@
+set hive.strict.checks.cartesian.product=false;
+set hive.materializedview.rewriting=true;
+set hive.stats.column.autogather=true;
+
+create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int);
+
+insert into cmv_basetable values
+ (1, 'alfred', 10.30, 2),
+ (2, 'bob', 3.14, 3),
+ (2, 'bonnie', 172342.2, 3),
+ (3, 'calvin', 978.76, 3),
+ (3, 'charlie', 9.8, 1);
+
+create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int);
+
+insert into cmv_basetable_2 values
+ (1, 'alfred', 10.30, 2),
+ (3, 'calvin', 978.76, 3);
+
+EXPLAIN
+CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS
+ SELECT cmv_basetable.a, cmv_basetable_2.c
+ FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+ WHERE cmv_basetable_2.c > 10.0
+ GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS
+ SELECT cmv_basetable.a, cmv_basetable_2.c
+ FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+ WHERE cmv_basetable_2.c > 10.0
+ GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+-- USE THE VIEW
+EXPLAIN
+SELECT cmv_basetable.a
+FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+SELECT cmv_basetable.a
+FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+insert into cmv_basetable_2 values
+ (3, 'charlie', 15.8, 1);
+
+-- TODO: CANNOT USE THE VIEW, IT IS OUTDATED
+EXPLAIN
+SELECT cmv_basetable.a
+FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+SELECT cmv_basetable.a
+FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+-- REBUILD
+EXPLAIN
+ALTER MATERIALIZED VIEW cmv_mat_view REBUILD;
+
+ALTER MATERIALIZED VIEW cmv_mat_view REBUILD;
+
+-- NOW IT CAN BE USED AGAIN
+EXPLAIN
+SELECT cmv_basetable.a
+FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+SELECT cmv_basetable.a
+FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a)
+WHERE cmv_basetable_2.c > 10.10
+GROUP BY cmv_basetable.a, cmv_basetable_2.c;
+
+drop materialized view cmv_mat_view;
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientnegative/create_view_failure1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/create_view_failure1.q.out b/ql/src/test/results/clientnegative/create_view_failure1.q.out
index bf149fc..2b9a324 100644
--- a/ql/src/test/results/clientnegative/create_view_failure1.q.out
+++ b/ql/src/test/results/clientnegative/create_view_failure1.q.out
@@ -15,4 +15,4 @@ PREHOOK: type: CREATEVIEW
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@xxx12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. AlreadyExistsException(message:Table xxx12 already exists)
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Table already exists: default.xxx12
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out b/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out
new file mode 100644
index 0000000..28f0a72
--- /dev/null
+++ b/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out
@@ -0,0 +1,40 @@
+PREHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@amvrng_table
+POSTHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@amvrng_table
+PREHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@amvrng_table
+POSTHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@amvrng_table
+POSTHOOK: Lineage: amvrng_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: amvrng_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: amvrng_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: grant select on table amvrng_table to user user2 with grant option
+PREHOOK: type: GRANT_PRIVILEGE
+PREHOOK: Output: default@amvrng_table
+POSTHOOK: query: grant select on table amvrng_table to user user2 with grant option
+POSTHOOK: type: GRANT_PRIVILEGE
+POSTHOOK: Output: default@amvrng_table
+PREHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@amvrng_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@amvrng_mat_view
+POSTHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@amvrng_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@amvrng_mat_view
+PREHOOK: query: revoke grant option for select on table amvrng_table from user user2
+PREHOOK: type: REVOKE_PRIVILEGE
+PREHOOK: Output: default@amvrng_table
+POSTHOOK: query: revoke grant option for select on table amvrng_table from user user2
+POSTHOOK: type: REVOKE_PRIVILEGE
+POSTHOOK: Output: default@amvrng_table
+FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvrng_table]]
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out b/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out
new file mode 100644
index 0000000..72244aa
--- /dev/null
+++ b/ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out
@@ -0,0 +1,28 @@
+PREHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@amvro_table
+POSTHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@amvro_table
+PREHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@amvro_table
+POSTHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@amvro_table
+POSTHOOK: Lineage: amvro_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: amvro_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: amvro_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@amvro_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@amvro_mat_view
+POSTHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@amvro_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@amvro_mat_view
+FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvro_table]]
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
index 81a7950..aa3240c 100644
--- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
@@ -52,6 +52,7 @@ PREHOOK: query: show tblproperties cmv_mat_view
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties cmv_mat_view
POSTHOOK: type: SHOW_TBLPROPERTIES
+COLUMN_STATS_ACCURATE {}
numFiles 1
totalSize 453
#### A masked pattern was here ####
@@ -75,12 +76,13 @@ POSTHOOK: query: select * from cmv_mat_view2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@cmv_mat_view2
#### A masked pattern was here ####
-3 978.76
3 9.80
+3 978.76
PREHOOK: query: show tblproperties cmv_mat_view2
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties cmv_mat_view2
POSTHOOK: type: SHOW_TBLPROPERTIES
+COLUMN_STATS_ACCURATE {}
numFiles 1
totalSize 322
#### A masked pattern was here ####
@@ -117,8 +119,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@cmv_basetable
POSTHOOK: Input: default@cmv_mat_view2
#### A masked pattern was here ####
-3 978.76
3 9.80
+3 978.76
PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
PREHOOK: Input: default@cmv_mat_view2
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientpositive/create_view.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_view.q.out b/ql/src/test/results/clientpositive/create_view.q.out
index 8763fdb..ffe69eb 100644
--- a/ql/src/test/results/clientpositive/create_view.q.out
+++ b/ql/src/test/results/clientpositive/create_view.q.out
@@ -168,12 +168,10 @@ STAGE PLANS:
Stage: Stage-1
Create View Operator:
Create View
- or replace: false
columns: valoo string
expanded text: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view0`
name: default.view0
original text: SELECT upper(value) FROM src WHERE key=86
- rewrite enabled: false
PREHOOK: query: EXPLAIN
SELECT * from view2 where key=18
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientpositive/create_view_translate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_view_translate.q.out b/ql/src/test/results/clientpositive/create_view_translate.q.out
index e574897..2365a74 100644
--- a/ql/src/test/results/clientpositive/create_view_translate.q.out
+++ b/ql/src/test/results/clientpositive/create_view_translate.q.out
@@ -135,12 +135,10 @@ STAGE PLANS:
Stage: Stage-1
Create View Operator:
Create View
- or replace: false
columns: id int, _c1 string
expanded text: SELECT `items`.`id`, `items`.`info`['price'] FROM `default`.`items`
name: default.priceview
original text: SELECT items.id, items.info['price'] FROM items
- rewrite enabled: false
PREHOOK: query: CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items
PREHOOK: type: CREATEVIEW
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientpositive/explain_ddl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/explain_ddl.q.out b/ql/src/test/results/clientpositive/explain_ddl.q.out
index f0e54c5..8a17a66 100644
--- a/ql/src/test/results/clientpositive/explain_ddl.q.out
+++ b/ql/src/test/results/clientpositive/explain_ddl.q.out
@@ -427,12 +427,10 @@ STAGE PLANS:
Stage: Stage-1
Create View Operator:
Create View
- or replace: false
columns: key string, value string
expanded text: select `m1`.`key`, `m1`.`value` from `default`.`M1`
name: default.V1
original text: select * from M1
- rewrite enabled: false
PREHOOK: query: EXPLAIN CREATE TABLE M1 LIKE src
PREHOOK: type: CREATETABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/a9e9d600/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
new file mode 100644
index 0000000..9286183
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
@@ -0,0 +1,207 @@
+PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_basetable
+POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_basetable
+PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@cmv_basetable
+POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@cmv_basetable
+POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view
+POSTHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view
+PREHOOK: query: select * from cmv_mat_view
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cmv_mat_view
+#### A masked pattern was here ####
+POSTHOOK: query: select * from cmv_mat_view
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cmv_mat_view
+#### A masked pattern was here ####
+1 alfred 10.30
+2 bob 3.14
+2 bonnie 172342.20
+3 calvin 978.76
+3 charlie 9.80
+PREHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view2
+POSTHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view2
+PREHOOK: query: select * from cmv_mat_view2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cmv_mat_view2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from cmv_mat_view2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cmv_mat_view2
+#### A masked pattern was here ####
+1 10.30
+2 3.14
+2 172342.20
+3 978.76
+3 9.80
+PREHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view3
+POSTHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view3
+PREHOOK: query: select * from cmv_mat_view3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cmv_mat_view3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from cmv_mat_view3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cmv_mat_view3
+#### A masked pattern was here ####
+2 bob 3.14
+2 bonnie 172342.20
+3 calvin 978.76
+3 charlie 9.80
+PREHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view4
+POSTHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view4
+PREHOOK: query: select * from cmv_mat_view4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cmv_mat_view4
+#### A masked pattern was here ####
+POSTHOOK: query: select * from cmv_mat_view4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cmv_mat_view4
+#### A masked pattern was here ####
+1 10.30
+2 172345.34
+3 988.56
+PREHOOK: query: describe extended cmv_mat_view4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@cmv_mat_view4
+POSTHOOK: query: describe extended cmv_mat_view4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@cmv_mat_view4
+a int
+_c1 decimal(20,2)
+
+#### A masked pattern was here ####
+PREHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_basetable2
+POSTHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_basetable2
+PREHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@cmv_basetable2
+POSTHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@cmv_basetable2
+POSTHOOK: Lineage: cmv_basetable2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: cmv_basetable2.e EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: cmv_basetable2.f EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e)
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable
+PREHOOK: Input: default@cmv_basetable2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view5
+POSTHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e)
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable
+POSTHOOK: Input: default@cmv_basetable2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view5
+PREHOOK: query: select * from cmv_mat_view5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cmv_mat_view5
+#### A masked pattern was here ####
+POSTHOOK: query: select * from cmv_mat_view5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cmv_mat_view5
+#### A masked pattern was here ####
+1 alfred 4 10.30 100.30
+2 bob 4 3.14 6133.00
+2 bonnie 5 172342.20 172.20
+3 calvin 6 978.76 8.76
+3 charlie 17 9.80 13144339.80
+PREHOOK: query: show tblproperties cmv_mat_view5
+PREHOOK: type: SHOW_TBLPROPERTIES
+POSTHOOK: query: show tblproperties cmv_mat_view5
+POSTHOOK: type: SHOW_TBLPROPERTIES
+COLUMN_STATS_ACCURATE {}
+key value
+numFiles 1
+totalSize 710
+#### A masked pattern was here ####
+PREHOOK: query: drop materialized view cmv_mat_view
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_mat_view
+PREHOOK: Output: default@cmv_mat_view
+POSTHOOK: query: drop materialized view cmv_mat_view
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_mat_view
+POSTHOOK: Output: default@cmv_mat_view
+PREHOOK: query: drop materialized view cmv_mat_view2
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_mat_view2
+PREHOOK: Output: default@cmv_mat_view2
+POSTHOOK: query: drop materialized view cmv_mat_view2
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_mat_view2
+POSTHOOK: Output: default@cmv_mat_view2
+PREHOOK: query: drop materialized view cmv_mat_view3
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_mat_view3
+PREHOOK: Output: default@cmv_mat_view3
+POSTHOOK: query: drop materialized view cmv_mat_view3
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_mat_view3
+POSTHOOK: Output: default@cmv_mat_view3
+PREHOOK: query: drop materialized view cmv_mat_view4
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_mat_view4
+PREHOOK: Output: default@cmv_mat_view4
+POSTHOOK: query: drop materialized view cmv_mat_view4
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_mat_view4
+POSTHOOK: Output: default@cmv_mat_view4
+PREHOOK: query: drop materialized view cmv_mat_view5
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_mat_view5
+PREHOOK: Output: default@cmv_mat_view5
+POSTHOOK: query: drop materialized view cmv_mat_view5
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_mat_view5
+POSTHOOK: Output: default@cmv_mat_view5