You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/08/09 03:34:02 UTC
hive git commit: HIVE-20336: Masking and filtering policies for
materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 3a649b6d0 -> 873d31f33
HIVE-20336: Masking and filtering policies for materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/873d31f3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/873d31f3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/873d31f3
Branch: refs/heads/master
Commit: 873d31f33a061cd38be7de91b208987871fb612e
Parents: 3a649b6
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed Aug 8 20:33:49 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed Aug 8 20:33:49 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 90 +-
.../apache/hadoop/hive/ql/parse/TableMask.java | 41 +-
ql/src/test/queries/clientnegative/masking_mv.q | 17 +
ql/src/test/queries/clientpositive/masking_mv.q | 77 +
.../results/clientnegative/masking_mv.q.out | 119 ++
.../results/clientpositive/masking_mv.q.out | 1349 ++++++++++++++++++
7 files changed, 1655 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 37bc153..0d74671 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -386,6 +386,9 @@ public enum ErrorMsg {
MASKING_FILTERING_ON_ACID_NOT_SUPPORTED(10287,
"Detected {0}.{1} has row masking/column filtering enabled, " +
"which is not supported for query involving ACID operations", true),
+ MASKING_FILTERING_ON_MATERIALIZED_VIEWS_SOURCES(10288,
+ "Querying directly materialized view contents is not supported since we detected {0}.{1} " +
+ "used by materialized view has row masking/column filtering enabled", true),
UPDATEDELETE_PARSE_ERROR(10290, "Encountered parse error while parsing rewritten merge/update or " +
"delete query"),
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 474c793..4638805 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -11930,30 +11930,36 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
continue;
}
- String replacementText = null;
- Table table = null;
- try {
- if (!tabNameToTabObject.containsKey(tabIdName)) {
- table = db.getTable(tabIdName, true);
- tabNameToTabObject.put(tabIdName, table);
- } else {
- table = tabNameToTabObject.get(tabIdName);
- }
- } catch (HiveException e) {
+ Table table = getTable(db, tabIdName, tabNameToTabObject);
+ if (table == null) {
// Table may not be found when materialization of CTE is on.
- STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
continue;
}
- List<String> colNames = new ArrayList<>();
- List<String> colTypes = new ArrayList<>();
- for (FieldSchema col : table.getAllCols()) {
- colNames.add(col.getName());
- colTypes.add(col.getType());
- }
+ if (table.isMaterializedView()) {
+ // When we are querying a materialized view directly, we check whether the source tables
+ // do not apply any policies.
+ for (String qName : table.getCreationMetadata().getTablesUsed()) {
+ table = getTable(db, qName, tabNameToTabObject);
+ if (table == null) {
+ // This should not happen
+ throw new SemanticException("Table " + qName + " not found when trying to obtain it to check masking/filtering " +
+ "policies");
+ }
- basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames),
- new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView(), table.isNonNative()));
+ List<String> colNames = new ArrayList<>();
+ extractColumnInfos(table, colNames, new ArrayList<>());
+
+ basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), null);
+ }
+ } else {
+ List<String> colNames = new ArrayList<>();
+ List<String> colTypes = new ArrayList<>();
+ extractColumnInfos(table, colNames, colTypes);
+
+ basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames),
+ new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView(), table.isNonNative()));
+ }
}
if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) {
for (Node child : astNode.getChildren()) {
@@ -11968,20 +11974,52 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) {
for (HivePrivilegeObject privObj : needRewritePrivObjs) {
MaskAndFilterInfo info = basicInfos.get(privObj);
- String replacementText = tableMask.create(privObj, info);
- if (replacementText != null) {
- // We don't support masking/filtering against ACID query at the moment
- if (ctx.getIsUpdateDeleteMerge()) {
- throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED,
+ // First we check whether entity actually needs masking or filtering
+ if (tableMask.needsMaskingOrFiltering(privObj)) {
+ if (info == null) {
+ // This is a table used by a materialized view
+ // Currently we do not support querying directly a materialized view
+ // when mask/filter should be applied on source tables
+ throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_MATERIALIZED_VIEWS_SOURCES,
privObj.getDbname(), privObj.getObjectName());
+ } else {
+ String replacementText = tableMask.create(privObj, info);
+ // We don't support masking/filtering against ACID query at the moment
+ if (ctx.getIsUpdateDeleteMerge()) {
+ throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED,
+ privObj.getDbname(), privObj.getObjectName());
+ }
+ tableMask.setNeedsRewrite(true);
+ tableMask.addTranslation(info.astNode, replacementText);
}
- tableMask.setNeedsRewrite(true);
- tableMask.addTranslation(info.astNode, replacementText);
}
}
}
}
+ private static Table getTable(Hive db, String tabIdName, Map<String, Table> tabNameToTabObject) {
+ Table table = null;
+ try {
+ if (!tabNameToTabObject.containsKey(tabIdName)) {
+ table = db.getTable(tabIdName, true);
+ tabNameToTabObject.put(tabIdName, table);
+ } else {
+ table = tabNameToTabObject.get(tabIdName);
+ }
+ } catch (HiveException e) {
+ // Table may not be found when materialization of CTE is on.
+ STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
+ }
+ return table;
+ }
+
+ private static void extractColumnInfos(Table table, List<String> colNames, List<String> colTypes) {
+ for (FieldSchema col : table.getAllCols()) {
+ colNames.add(col.getName());
+ colTypes.add(col.getType());
+ }
+ }
+
// We walk through the AST.
// We replace all the TOK_TABREF by adding additional masking and filter if
// the table needs to be masked or filtered.
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java
index ee93cf6..b1db547 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java
@@ -83,26 +83,43 @@ public class TableMask {
return authorizer.needTransform();
}
- public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo)
+ public boolean needsMaskingOrFiltering(HivePrivilegeObject privObject)
throws SemanticException {
- boolean doColumnMasking = false;
- boolean doRowFiltering = false;
- StringBuilder sb = new StringBuilder();
- sb.append("(SELECT ");
- boolean firstOne = true;
+ String filter = privObject.getRowFilterExpression();
+ if (filter != null) {
+ return true;
+ }
List<String> exprs = privObject.getCellValueTransformers();
if (exprs != null) {
if (exprs.size() != privObject.getColumns().size()) {
throw new SemanticException("Expect " + privObject.getColumns().size() + " columns in "
+ privObject.getObjectName() + ", but only find " + exprs.size());
}
- List<String> colTypes = maskAndFilterInfo.colTypes;
for (int index = 0; index < exprs.size(); index++) {
String expr = exprs.get(index);
if (expr == null) {
throw new SemanticException("Expect string type CellValueTransformer in "
+ privObject.getObjectName() + ", but only find null");
}
+ String colName = privObject.getColumns().get(index);
+ if (!expr.equals(colName)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) {
+ boolean doColumnMasking = false;
+ StringBuilder sb = new StringBuilder();
+ sb.append("(SELECT ");
+ boolean firstOne = true;
+ List<String> exprs = privObject.getCellValueTransformers();
+ if (exprs != null) {
+ List<String> colTypes = maskAndFilterInfo.colTypes;
+ for (int index = 0; index < exprs.size(); index++) {
+ String expr = exprs.get(index);
if (!firstOne) {
sb.append(", ");
} else {
@@ -141,17 +158,11 @@ public class TableMask {
String filter = privObject.getRowFilterExpression();
if (filter != null) {
sb.append(" WHERE " + filter);
- doRowFiltering = true;
}
sb.append(")" + HiveUtils.unparseIdentifier(maskAndFilterInfo.alias, conf));
- if (!doColumnMasking && !doRowFiltering) {
- // nothing to do
- return null;
- } else {
- LOG.debug("TableMask creates `" + sb.toString() + "`");
- return sb.toString();
- }
+ LOG.debug("TableMask creates `" + sb.toString() + "`");
+ return sb.toString();
}
void addTranslation(ASTNode node, String replacementText) throws SemanticException {
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/test/queries/clientnegative/masking_mv.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/masking_mv.q b/ql/src/test/queries/clientnegative/masking_mv.q
new file mode 100644
index 0000000..deb5c4a
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/masking_mv.q
@@ -0,0 +1,17 @@
+--! qt:dataset:srcpart
+--! qt:dataset:src
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src;
+
+explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`;
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`;
+
+select key from `masking_test_view_n_mv`;
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/test/queries/clientpositive/masking_mv.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/masking_mv.q b/ql/src/test/queries/clientpositive/masking_mv.q
new file mode 100644
index 0000000..847b429
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/masking_mv.q
@@ -0,0 +1,77 @@
+--! qt:dataset:srcpart
+--! qt:dataset:src
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src;
+
+explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`;
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`;
+describe formatted `masking_test_view_n_mv`;
+
+-- This will use the MV and will apply masking/filtering on top
+-- of the source table, not any policy that applies on the MV
+explain
+select key from `masking_test_n_mv`;
+select key from `masking_test_n_mv`;
+
+create materialized view `masking_test_view_n_mv_3` as
+select value, sum(key) from `masking_test_n_mv` group by value;
+
+-- Rewriting triggered from `masking_test_view_n_mv`
+explain
+select key from `masking_test_n_mv` group by key;
+select key from `masking_test_n_mv` group by key;
+
+-- Rewriting cannot be triggered
+explain
+select value from `masking_test_n_mv` group by value;
+select value from `masking_test_n_mv` group by value;
+
+-- Rewriting cannot be triggered
+explain
+select value, sum(key) from `masking_test_n_mv` group by value;
+select value, sum(key) from `masking_test_n_mv` group by value;
+
+create materialized view `masking_test_view_n_mv_4` as
+select key, value from `masking_test_n_mv`;
+
+-- Rewriting triggered from `masking_test_view_n_mv_4`
+explain
+select value from `masking_test_n_mv` group by value;
+select value from `masking_test_n_mv` group by value;
+
+-- Rewriting triggered from `masking_test_view_n_mv_4`
+explain
+select value, sum(key) from `masking_test_n_mv` group by value;
+select value, sum(key) from `masking_test_n_mv` group by value;
+
+
+create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src;
+
+explain
+create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`;
+create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`;
+describe formatted `masking_test_view_n_mv_2`;
+
+-- This is allowed because source tables do not use a masking
+-- or filtering. Policy is not applied on MV as expected
+explain
+select key from `masking_test_view_n_mv_2` order by key;
+select key from `masking_test_view_n_mv_2` order by key;
+
+drop materialized view `masking_test_view_n_mv`;
+drop materialized view `masking_test_view_n_mv_2`;
+drop materialized view `masking_test_view_n_mv_3`;
+drop materialized view `masking_test_view_n_mv_4`;
+drop table `masking_test_n_mv`;
+drop table `srcTnx`;
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/test/results/clientnegative/masking_mv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/masking_mv.q.out b/ql/src/test/results/clientnegative/masking_mv.q.out
new file mode 100644
index 0000000..1c40f50
--- /dev/null
+++ b/ql/src/test/results/clientnegative/masking_mv.q.out
@@ -0,0 +1,119 @@
+PREHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_n_mv
+POSTHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_n_mv
+POSTHOOK: Lineage: masking_test_n_mv.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: masking_test_n_mv.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: query: explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-8 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-8
+ Stage-9 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test_n_mv
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.masking_test_view_n_mv
+ Execution mode: vectorized
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-8
+ Create View Operator:
+ Create View
+ columns: key int
+ expanded text: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv`
+ name: default.masking_test_view_n_mv
+ original text: select key from masking_test_n_mv
+ rewrite enabled: true
+
+ Stage: Stage-2
+ Stats Work
+ Basic Stats Work:
+
+ Stage: Stage-9
+ Materialized View Work
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_view_n_mv
+POSTHOOK: query: create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_view_n_mv
+FAILED: SemanticException [Error 10288]: Querying directly materialized view contents is not supported since we detected default.masking_test_n_mv used by materialized view has row masking/column filtering enabled
http://git-wip-us.apache.org/repos/asf/hive/blob/873d31f3/ql/src/test/results/clientpositive/masking_mv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/masking_mv.q.out b/ql/src/test/results/clientpositive/masking_mv.q.out
new file mode 100644
index 0000000..08c2516
--- /dev/null
+++ b/ql/src/test/results/clientpositive/masking_mv.q.out
@@ -0,0 +1,1349 @@
+PREHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_n_mv
+POSTHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_n_mv
+POSTHOOK: Lineage: masking_test_n_mv.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: masking_test_n_mv.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: query: explain
+create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-8 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-8
+ Stage-9 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test_n_mv
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.masking_test_view_n_mv
+ Execution mode: vectorized
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-8
+ Create View Operator:
+ Create View
+ columns: key int
+ expanded text: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv`
+ name: default.masking_test_view_n_mv
+ original text: select key from masking_test_n_mv
+ rewrite enabled: true
+
+ Stage: Stage-2
+ Stats Work
+ Basic Stats Work:
+
+ Stage: Stage-9
+ Materialized View Work
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_view_n_mv
+POSTHOOK: query: create materialized view `masking_test_view_n_mv` as
+select key from `masking_test_n_mv`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_view_n_mv
+PREHOOK: query: describe formatted `masking_test_view_n_mv`
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@masking_test_view_n_mv
+POSTHOOK: query: describe formatted `masking_test_view_n_mv`
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@masking_test_view_n_mv
+# col_name data_type comment
+key int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MATERIALIZED_VIEW
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
+ numFiles 1
+ numRows 500
+ rawDataSize 2000
+ totalSize 974
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# Materialized View Information
+Original Query: select key from masking_test_n_mv
+Expanded Query: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv`
+Rewrite Enabled: Yes
+Outdated for Rewriting: No
+PREHOOK: query: explain
+select key from `masking_test_n_mv`
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key from `masking_test_n_mv`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: default.masking_test_view_n_mv
+ filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((0 = (key % 2)) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key from `masking_test_n_mv`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv
+#### A masked pattern was here ####
+POSTHOOK: query: select key from `masking_test_n_mv`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv
+#### A masked pattern was here ####
+0
+4
+8
+0
+0
+2
+PREHOOK: query: create materialized view `masking_test_view_n_mv_3` as
+select value, sum(key) from `masking_test_n_mv` group by value
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_view_n_mv_3
+POSTHOOK: query: create materialized view `masking_test_view_n_mv_3` as
+select value, sum(key) from `masking_test_n_mv` group by value
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_view_n_mv_3
+PREHOOK: query: explain
+select key from `masking_test_n_mv` group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key from `masking_test_n_mv` group by key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: default.masking_test_view_n_mv
+ filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((0 = (key % 2)) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 41 Data size: 164 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 164 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key from `masking_test_n_mv` group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv
+#### A masked pattern was here ####
+POSTHOOK: query: select key from `masking_test_n_mv` group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv
+#### A masked pattern was here ####
+0
+2
+4
+8
+PREHOOK: query: explain
+select value from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select value from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test_n_mv
+ filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 41 Data size: 2476 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 2476 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select value from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv_3
+#### A masked pattern was here ####
+POSTHOOK: query: select value from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv_3
+#### A masked pattern was here ####
+0_lav
+2_lav
+4_lav
+8_lav
+PREHOOK: query: explain
+select value, sum(key) from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select value, sum(key) from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test_n_mv
+ filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string), key (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 5013 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 41 Data size: 2476 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 2476 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv_3
+#### A masked pattern was here ####
+POSTHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv_3
+#### A masked pattern was here ####
+0_lav 0
+2_lav 2
+4_lav 4
+8_lav 8
+PREHOOK: query: create materialized view `masking_test_view_n_mv_4` as
+select key, value from `masking_test_n_mv`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_view_n_mv_4
+POSTHOOK: query: create materialized view `masking_test_view_n_mv_4` as
+select key, value from `masking_test_n_mv`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_view_n_mv_4
+PREHOOK: query: explain
+select value from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select value from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: default.masking_test_view_n_mv_4
+ filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
+ Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((0 = (key % 2)) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 41 Data size: 3854 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 3854 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select value from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv_3
+PREHOOK: Input: default@masking_test_view_n_mv_4
+#### A masked pattern was here ####
+POSTHOOK: query: select value from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv_3
+POSTHOOK: Input: default@masking_test_view_n_mv_4
+#### A masked pattern was here ####
+0_lav
+2_lav
+4_lav
+8_lav
+PREHOOK: query: explain
+select value, sum(key) from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select value, sum(key) from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: default.masking_test_view_n_mv_4
+ filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean)
+ Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((0 = (key % 2)) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string), key (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 7802 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 41 Data size: 3854 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 3854 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Input: default@masking_test_view_n_mv_3
+PREHOOK: Input: default@masking_test_view_n_mv_4
+#### A masked pattern was here ####
+POSTHOOK: query: select value, sum(key) from `masking_test_n_mv` group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Input: default@masking_test_view_n_mv_3
+POSTHOOK: Input: default@masking_test_view_n_mv_4
+#### A masked pattern was here ####
+0_lav 0
+2_lav 2
+4_lav 4
+8_lav 8
+PREHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcTnx
+POSTHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as
+select cast(key as int) as key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcTnx
+POSTHOOK: Lineage: srctnx.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srctnx.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: explain
+create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: query: explain
+create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-8 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-8
+ Stage-9 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srctnx
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.masking_test_view_n_mv_2
+ Execution mode: vectorized
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-8
+ Create View Operator:
+ Create View
+ columns: key int
+ expanded text: select `srctnx`.`key` from `default`.`srcTnx`
+ name: default.masking_test_view_n_mv_2
+ original text: select key from srcTnx
+ rewrite enabled: true
+
+ Stage: Stage-2
+ Stats Work
+ Basic Stats Work:
+
+ Stage: Stage-9
+ Materialized View Work
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@srctnx
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test_view_n_mv_2
+POSTHOOK: query: create materialized view `masking_test_view_n_mv_2` as
+select key from `srcTnx`
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@srctnx
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test_view_n_mv_2
+PREHOOK: query: describe formatted `masking_test_view_n_mv_2`
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@masking_test_view_n_mv_2
+POSTHOOK: query: describe formatted `masking_test_view_n_mv_2`
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@masking_test_view_n_mv_2
+# col_name data_type comment
+key int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MATERIALIZED_VIEW
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
+ numFiles 1
+ numRows 500
+ rawDataSize 2000
+ totalSize 974
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# Materialized View Information
+Original Query: select key from srcTnx
+Expanded Query: select `srctnx`.`key` from `default`.`srcTnx`
+Rewrite Enabled: Yes
+Outdated for Rewriting: No
+PREHOOK: query: explain
+select key from `masking_test_view_n_mv_2` order by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key from `masking_test_view_n_mv_2` order by key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test_view_n_mv_2
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key from `masking_test_view_n_mv_2` order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test_view_n_mv_2
+#### A masked pattern was here ####
+POSTHOOK: query: select key from `masking_test_view_n_mv_2` order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test_view_n_mv_2
+#### A masked pattern was here ####
+0
+0
+0
+2
+4
+5
+5
+5
+8
+9
+10
+11
+12
+12
+15
+15
+17
+18
+18
+19
+20
+24
+24
+26
+26
+27
+28
+30
+33
+34
+35
+35
+35
+37
+37
+41
+42
+42
+43
+44
+47
+51
+51
+53
+54
+57
+58
+58
+64
+65
+66
+67
+67
+69
+70
+70
+70
+72
+72
+74
+76
+76
+77
+78
+80
+82
+83
+83
+84
+84
+85
+86
+87
+90
+90
+90
+92
+95
+95
+96
+97
+97
+98
+98
+100
+100
+103
+103
+104
+104
+105
+111
+113
+113
+114
+116
+118
+118
+119
+119
+119
+120
+120
+125
+125
+126
+128
+128
+128
+129
+129
+131
+133
+134
+134
+136
+137
+137
+138
+138
+138
+138
+143
+145
+146
+146
+149
+149
+150
+152
+152
+153
+155
+156
+157
+158
+160
+162
+163
+164
+164
+165
+165
+166
+167
+167
+167
+168
+169
+169
+169
+169
+170
+172
+172
+174
+174
+175
+175
+176
+176
+177
+178
+179
+179
+180
+181
+183
+186
+187
+187
+187
+189
+190
+191
+191
+192
+193
+193
+193
+194
+195
+195
+196
+197
+197
+199
+199
+199
+200
+200
+201
+202
+203
+203
+205
+205
+207
+207
+208
+208
+208
+209
+209
+213
+213
+214
+216
+216
+217
+217
+218
+219
+219
+221
+221
+222
+223
+223
+224
+224
+226
+228
+229
+229
+230
+230
+230
+230
+230
+233
+233
+235
+237
+237
+238
+238
+239
+239
+241
+242
+242
+244
+247
+248
+249
+252
+255
+255
+256
+256
+257
+258
+260
+262
+263
+265
+265
+266
+272
+272
+273
+273
+273
+274
+275
+277
+277
+277
+277
+278
+278
+280
+280
+281
+281
+282
+282
+283
+284
+285
+286
+287
+288
+288
+289
+291
+292
+296
+298
+298
+298
+302
+305
+306
+307
+307
+308
+309
+309
+310
+311
+311
+311
+315
+316
+316
+316
+317
+317
+318
+318
+318
+321
+321
+322
+322
+323
+325
+325
+327
+327
+327
+331
+331
+332
+333
+333
+335
+336
+338
+339
+341
+342
+342
+344
+344
+345
+348
+348
+348
+348
+348
+351
+353
+353
+356
+360
+362
+364
+365
+366
+367
+367
+368
+369
+369
+369
+373
+374
+375
+377
+378
+379
+382
+382
+384
+384
+384
+386
+389
+392
+393
+394
+395
+395
+396
+396
+396
+397
+397
+399
+399
+400
+401
+401
+401
+401
+401
+402
+403
+403
+403
+404
+404
+406
+406
+406
+406
+407
+409
+409
+409
+411
+413
+413
+414
+414
+417
+417
+417
+418
+419
+421
+424
+424
+427
+429
+429
+430
+430
+430
+431
+431
+431
+432
+435
+436
+437
+438
+438
+438
+439
+439
+443
+444
+446
+448
+449
+452
+453
+454
+454
+454
+455
+457
+458
+458
+459
+459
+460
+462
+462
+463
+463
+466
+466
+466
+467
+468
+468
+468
+468
+469
+469
+469
+469
+469
+470
+472
+475
+477
+478
+478
+479
+480
+480
+480
+481
+482
+483
+484
+485
+487
+489
+489
+489
+489
+490
+491
+492
+492
+493
+494
+495
+496
+497
+498
+498
+498
+PREHOOK: query: drop materialized view `masking_test_view_n_mv`
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_view_n_mv
+PREHOOK: Output: default@masking_test_view_n_mv
+POSTHOOK: query: drop materialized view `masking_test_view_n_mv`
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_view_n_mv
+POSTHOOK: Output: default@masking_test_view_n_mv
+PREHOOK: query: drop materialized view `masking_test_view_n_mv_2`
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_view_n_mv_2
+PREHOOK: Output: default@masking_test_view_n_mv_2
+POSTHOOK: query: drop materialized view `masking_test_view_n_mv_2`
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_view_n_mv_2
+POSTHOOK: Output: default@masking_test_view_n_mv_2
+PREHOOK: query: drop materialized view `masking_test_view_n_mv_3`
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_view_n_mv_3
+PREHOOK: Output: default@masking_test_view_n_mv_3
+POSTHOOK: query: drop materialized view `masking_test_view_n_mv_3`
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_view_n_mv_3
+POSTHOOK: Output: default@masking_test_view_n_mv_3
+PREHOOK: query: drop materialized view `masking_test_view_n_mv_4`
+PREHOOK: type: DROP_MATERIALIZED_VIEW
+PREHOOK: Input: default@masking_test_view_n_mv_4
+PREHOOK: Output: default@masking_test_view_n_mv_4
+POSTHOOK: query: drop materialized view `masking_test_view_n_mv_4`
+POSTHOOK: type: DROP_MATERIALIZED_VIEW
+POSTHOOK: Input: default@masking_test_view_n_mv_4
+POSTHOOK: Output: default@masking_test_view_n_mv_4
+PREHOOK: query: drop table `masking_test_n_mv`
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@masking_test_n_mv
+PREHOOK: Output: default@masking_test_n_mv
+POSTHOOK: query: drop table `masking_test_n_mv`
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@masking_test_n_mv
+POSTHOOK: Output: default@masking_test_n_mv
+PREHOOK: query: drop table `srcTnx`
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srctnx
+PREHOOK: Output: default@srctnx
+POSTHOOK: query: drop table `srcTnx`
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srctnx
+POSTHOOK: Output: default@srctnx