You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2024/03/11 08:11:10 UTC
(hive) branch master updated: HIVE-28098: Fails to copy empty column statistics of materialized CTE (okumin, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new aded1821cb2 HIVE-28098: Fails to copy empty column statistics of materialized CTE (okumin, reviewed by Krisztian Kasa)
aded1821cb2 is described below
commit aded1821cb27bfe110c556476852334b21e0028a
Author: okumin <gi...@okumin.com>
AuthorDate: Mon Mar 11 17:11:04 2024 +0900
HIVE-28098: Fails to copy empty column statistics of materialized CTE (okumin, reviewed by Krisztian Kasa)
---
.../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 43 +-
ql/src/test/queries/clientpositive/cte_mat_11.q | 166 ++
.../results/clientpositive/llap/cte_mat_11.q.out | 1646 ++++++++++++++++++++
4 files changed, 1841 insertions(+), 16 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 40d9167c74f..3f49e1098da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1071,7 +1071,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
LOG.info(cteName + " will be materialized into " + location);
cte.source = analyzer;
- ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp(), table));
+ ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp()));
return table;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 227baef5ce4..b15495a67dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -1600,26 +1600,39 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
LOG.info("{} will be materialized into {}", cteName, location);
cte.source = analyzer;
- ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp(), table));
+ ctx.addMaterializedTable(cteName, table, getMaterializedTableStats(analyzer.getSinkOp()));
return table;
}
- static Statistics getMaterializedTableStats(Operator<?> sinkOp, Table table) {
+ protected Statistics getMaterializedTableStats(Operator<?> sinkOp) {
final Statistics tableStats = sinkOp.getStatistics().clone();
- final List<ColStatistics> sourceColStatsList = tableStats.getColumnStats();
- final List<String> colNames = table.getCols().stream().map(FieldSchema::getName).collect(Collectors.toList());
- if (sourceColStatsList.size() != colNames.size()) {
- throw new IllegalStateException(String.format(
- "The size of col stats must be equal to that of schema. Stats = %s, Schema = %s",
- sourceColStatsList, colNames));
- }
- final List<ColStatistics> colStatsList = new ArrayList<>(sourceColStatsList.size());
- for (int i = 0; i < sourceColStatsList.size(); i++) {
- final ColStatistics colStats = sourceColStatsList.get(i);
- // FileSinkOperator stores column stats with internal names such as "_col1"
- colStats.setColumnName(colNames.get(i));
- colStatsList.add(colStats);
+ if (tableStats.getColumnStatsState() == Statistics.State.NONE || sinkOp.getNumParent() == 0) {
+ return tableStats;
+ }
+
+ final List<String> parentColumnNames = sinkOp.getParentOperators().get(0).getSchema().getColumnNames();
+ final List<String> childColumnNames = sinkOp.getSchema().getColumnNames();
+ if (parentColumnNames.size() != childColumnNames.size()) {
+ LOG.warn("The number of columns of FileSinkOperator is inconsistent. Parent = {}, Child = {}",
+ parentColumnNames, childColumnNames);
+ tableStats.setColumnStatsState(Statistics.State.NONE);
+ return tableStats;
+ }
+ final Map<String, String> mapping = new HashMap<>(parentColumnNames.size());
+ for (int i = 0; i < parentColumnNames.size(); i++) {
+ mapping.put(parentColumnNames.get(i), childColumnNames.get(i));
+ }
+
+ final List<ColStatistics> colStatsList = tableStats.getColumnStats();
+ if (!mapping.keySet().equals(colStatsList.stream().map(ColStatistics::getColumnName).collect(Collectors.toSet()))) {
+ LOG.warn("The column statistics are inconsistent with the expected column names. Actual = {}, Expected = {}",
+ colStatsList, parentColumnNames);
+ tableStats.setColumnStatsState(Statistics.State.NONE);
+ return tableStats;
+ }
+ for (ColStatistics colStats : colStatsList) {
+ colStats.setColumnName(mapping.get(colStats.getColumnName()));
}
tableStats.setColumnStats(colStatsList);
return tableStats;
diff --git a/ql/src/test/queries/clientpositive/cte_mat_11.q b/ql/src/test/queries/clientpositive/cte_mat_11.q
index f1ace684b45..9b912731263 100644
--- a/ql/src/test/queries/clientpositive/cte_mat_11.q
+++ b/ql/src/test/queries/clientpositive/cte_mat_11.q
@@ -2,6 +2,10 @@
set hive.optimize.cte.materialize.threshold=2;
set hive.optimize.cte.materialize.full.aggregate.only=false;
+-- Test cases with full stats
+DESCRIBE FORMATTED src key;
+DESCRIBE FORMATTED src value;
+
EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src
),
@@ -49,3 +53,165 @@ materialized_cte2 AS (
SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2;
+
+-- Test cases with no stats
+set hive.stats.autogather=false;
+CREATE TABLE src_no_stats AS SELECT * FROM src;
+DESCRIBE FORMATTED src_no_stats key;
+DESCRIBE FORMATTED src_no_stats value;
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
+
+-- Test cases with partial stats(only key)
+CREATE TABLE src_partial_stats_key AS SELECT * FROM src;
+ANALYZE TABLE src_partial_stats_key COMPUTE STATISTICS FOR COLUMNS key;
+DESCRIBE FORMATTED src_partial_stats_key key;
+DESCRIBE FORMATTED src_partial_stats_key value;
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
+
+-- Test cases with partial stats(only value)
+CREATE TABLE src_partial_stats_value AS SELECT * FROM src;
+ANALYZE TABLE src_partial_stats_value COMPUTE STATISTICS FOR COLUMNS value;
+DESCRIBE FORMATTED src_partial_stats_value key;
+DESCRIBE FORMATTED src_partial_stats_value value;
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key);
+
+EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
+
+EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2;
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
index e48d6e834ed..e7ca9fa180f 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
@@ -1,3 +1,41 @@
+PREHOOK: query: DESCRIBE FORMATTED src key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src
+POSTHOOK: query: DESCRIBE FORMATTED src key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src
+col_name key
+data_type string
+min
+max
+num_nulls 0
+distinct_count 316
+avg_col_len 2.812
+max_col_len 3
+num_trues
+num_falses
+bit_vector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED src value
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src
+POSTHOOK: query: DESCRIBE FORMATTED src value
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src
+col_name value
+data_type string
+min
+max
+num_nulls 0
+distinct_count 307
+avg_col_len 6.812
+max_col_len 7
+num_trues
+num_falses
+bit_vector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src
),
@@ -478,3 +516,1611 @@ HiveUnion(all=[true])
HiveProject(key=[$0], value=[$1])
HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+PREHOOK: query: CREATE TABLE src_no_stats AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_no_stats
+POSTHOOK: query: CREATE TABLE src_no_stats AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_no_stats
+POSTHOOK: Lineage: src_no_stats.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_no_stats.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE FORMATTED src_no_stats key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_no_stats
+POSTHOOK: query: DESCRIBE FORMATTED src_no_stats key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_no_stats
+col_name key
+data_type string
+min
+max
+num_nulls
+distinct_count
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector
+comment from deserializer
+PREHOOK: query: DESCRIBE FORMATTED src_no_stats value
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_no_stats
+POSTHOOK: query: DESCRIBE FORMATTED src_no_stats value
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_no_stats
+col_name value
+data_type string
+min
+max
+num_nulls
+distinct_count
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector
+comment from deserializer
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_no_stats
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 121 Data size: 40643 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 133 Data size: 44707 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 146 Data size: 49177 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 146 Data size: 49177 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(key=[$0])
+ HiveJoin(condition=[=($0, $1)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[a])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[b])
+
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_no_stats
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 2 <- Union 3 (CONTAINS)
+ Map 4 <- Union 3 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 149 Data size: 50048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 3
+ Vertex: Union 3
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 5 <- Union 6 (CONTAINS)
+ Map 7 <- Union 6 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 596 Data size: 200192 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 298 Data size: 100096 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 596 Data size: 200192 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 6
+ Vertex: Union 6
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_no_stats
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveUnion(all=[true])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+
+PREHOOK: query: CREATE TABLE src_partial_stats_key AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_partial_stats_key
+POSTHOOK: query: CREATE TABLE src_partial_stats_key AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_partial_stats_key
+POSTHOOK: Lineage: src_partial_stats_key.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_partial_stats_key.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ANALYZE TABLE src_partial_stats_key COMPUTE STATISTICS FOR COLUMNS key
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@src_partial_stats_key
+PREHOOK: Output: default@src_partial_stats_key
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE src_partial_stats_key COMPUTE STATISTICS FOR COLUMNS key
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@src_partial_stats_key
+POSTHOOK: Output: default@src_partial_stats_key
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED src_partial_stats_key key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_partial_stats_key
+POSTHOOK: query: DESCRIBE FORMATTED src_partial_stats_key key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_partial_stats_key
+col_name key
+data_type string
+min
+max
+num_nulls 0
+distinct_count 316
+avg_col_len 2.812
+max_col_len 3
+num_trues
+num_falses
+bit_vector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED src_partial_stats_key value
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_partial_stats_key
+POSTHOOK: query: DESCRIBE FORMATTED src_partial_stats_key value
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_partial_stats_key
+col_name value
+data_type string
+min
+max
+num_nulls
+distinct_count
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_partial_stats_key
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 126484 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 126484 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 791 Data size: 68817 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1980 Data size: 172260 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1980 Data size: 172260 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(key=[$0])
+ HiveJoin(condition=[=($0, $1)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[a])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[b])
+
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_partial_stats_key
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 2 <- Union 3 (CONTAINS)
+ Map 4 <- Union 3 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 126484 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 252968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 500 Data size: 131084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 126484 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 252968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 3
+ Vertex: Union 3
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 5 <- Union 6 (CONTAINS)
+ Map 7 <- Union 6 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 1000 Data size: 252968 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 215984 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2000 Data size: 431968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 1000 Data size: 252968 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 215984 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2000 Data size: 431968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 6
+ Vertex: Union 6
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_key
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveUnion(all=[true])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+
+PREHOOK: query: CREATE TABLE src_partial_stats_value AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_partial_stats_value
+POSTHOOK: query: CREATE TABLE src_partial_stats_value AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_partial_stats_value
+POSTHOOK: Lineage: src_partial_stats_value.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_partial_stats_value.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ANALYZE TABLE src_partial_stats_value COMPUTE STATISTICS FOR COLUMNS value
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@src_partial_stats_value
+PREHOOK: Output: default@src_partial_stats_value
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE src_partial_stats_value COMPUTE STATISTICS FOR COLUMNS value
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@src_partial_stats_value
+POSTHOOK: Output: default@src_partial_stats_value
+#### A masked pattern was here ####
+PREHOOK: query: DESCRIBE FORMATTED src_partial_stats_value key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_partial_stats_value
+POSTHOOK: query: DESCRIBE FORMATTED src_partial_stats_value key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_partial_stats_value
+col_name key
+data_type string
+min
+max
+num_nulls
+distinct_count
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED src_partial_stats_value value
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_partial_stats_value
+POSTHOOK: query: DESCRIBE FORMATTED src_partial_stats_value value
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_partial_stats_value
+col_name value
+data_type string
+min
+max
+num_nulls 0
+distinct_count 307
+avg_col_len 6.812
+max_col_len 7
+num_trues
+num_falses
+bit_vector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}}
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_partial_stats_value
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 450 Data size: 115654 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 450 Data size: 74704 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 450 Data size: 74704 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 450 Data size: 115654 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 450 Data size: 74704 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 450 Data size: 74704 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2025 Data size: 364504 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2025 Data size: 364504 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 2025 Data size: 364504 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 2025 Data size: 364504 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1935 Data size: 340400 Basic stats: COMPLETE Column stats: PARTIAL
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 37442 Data size: 6873688 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 37442 Data size: 6873688 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT a.key
+ FROM materialized_cte1 a
+ JOIN materialized_cte1 b ON (a.key = b.key)
+)
+SELECT a.key
+FROM materialized_cte2 a
+JOIN materialized_cte2 b ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(key=[$0])
+ HiveJoin(condition=[=($0, $1)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[a])
+ HiveProject(key=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[b])
+
+PREHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-4 depends on stages: Stage-2, Stage-0
+ Stage-5 depends on stages: Stage-4
+ Stage-7 depends on stages: Stage-5, Stage-3
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-1
+ Stage-6 depends on stages: Stage-7
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_partial_stats_value
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 2 <- Union 3 (CONTAINS)
+ Map 4 <- Union 3 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 128484 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 256968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte1
+ Statistics: Num rows: 500 Data size: 133084 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 128484 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 256968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.materialized_cte2
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 3
+ Vertex: Union 3
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-7
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 5 <- Union 6 (CONTAINS)
+ Map 7 <- Union 6 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 1000 Data size: 256968 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 219984 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2000 Data size: 439968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: materialized_cte2
+ Statistics: Num rows: 1000 Data size: 256968 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 219984 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2000 Data size: 439968 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Union 6
+ Vertex: Union 6
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-6
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
+ SELECT * FROM src_partial_stats_value
+),
+materialized_cte2 AS (
+ SELECT * FROM materialized_cte1
+ UNION ALL
+ SELECT * FROM materialized_cte1
+)
+SELECT * FROM materialized_cte2
+UNION ALL
+SELECT * FROM materialized_cte2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveUnion(all=[true])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+ HiveProject(key=[$0], value=[$1])
+ HiveTableScan(table=[[default, materialized_cte2]], table:alias=[materialized_cte2])
+