You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/02/03 20:03:59 UTC
[34/51] [partial] hive git commit: HIVE-11394: Enhance EXPLAIN
display for vectorization (Matt McCline, reviewed by Gopal Vijayaraghavan)
http://git-wip-us.apache.org/repos/asf/hive/blob/fcb57100/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
index bc37741..b51bed6 100644
--- a/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out
@@ -38,47 +38,113 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@tbl2
POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from (
select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
) subq1
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from (
select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
) subq1
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=11 width=93)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from (
select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
@@ -95,7 +161,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
22
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(
select key, count(*) from
@@ -105,7 +171,7 @@ select count(*) from
group by key
) subq2
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(
select key, count(*) from
@@ -115,46 +181,153 @@ select count(*) from
group by key
) subq2
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0
+ native: false
+ projectedOutputColumns: []
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: []
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ aggregators: VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 3 vectorized, llap
- File Output Operator [FS_31]
- Group By Operator [GBY_30] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_29]
- Group By Operator [GBY_28] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_27] (rows=5 width=93)
- Group By Operator [GBY_26] (rows=5 width=93)
- Output:["_col0"],keys:KEY._col0
- <-Map 1 [SIMPLE_EDGE] llap
- SHUFFLE [RS_11]
- PartitionCols:_col0
- Group By Operator [GBY_10] (rows=11 width=93)
- Output:["_col0"],keys:_col0
- Merge Join Operator [MERGEJOIN_24] (rows=11 width=93)
- Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"]
- <-Select Operator [SEL_5] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_23] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_22] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(
@@ -181,7 +354,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
6
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select src1.key, src1.cnt1, src2.cnt1 from
(
select key, count(*) as cnt1 from
@@ -198,7 +371,7 @@ join
) src2
on src1.key = src2.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select src1.key, src1.cnt1, src2.cnt1 from
(
select key, count(*) as cnt1 from
@@ -215,71 +388,214 @@ join
) src2
on src1.key = src2.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
-Reducer 6 <- Map 5 (SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 3 llap
- File Output Operator [FS_32]
- Select Operator [SEL_31] (rows=5 width=102)
- Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_49] (rows=5 width=102)
- Conds:RS_51._col0=RS_53._col0(Inner),Output:["_col0","_col1","_col3"]
- <-Reducer 2 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_51]
- PartitionCols:_col0
- Group By Operator [GBY_50] (rows=5 width=93)
- Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
- <-Map 1 [SIMPLE_EDGE] llap
- SHUFFLE [RS_11]
- PartitionCols:_col0
- Group By Operator [GBY_10] (rows=11 width=93)
- Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
- Merge Join Operator [MERGEJOIN_45] (rows=11 width=93)
- Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"]
- <-Select Operator [SEL_5] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_42] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_41] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Reducer 6 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_53]
- PartitionCols:_col0
- Group By Operator [GBY_52] (rows=5 width=93)
- Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
- <-Map 5 [SIMPLE_EDGE] llap
- SHUFFLE [RS_25]
- PartitionCols:_col0
- Group By Operator [GBY_24] (rows=11 width=93)
- Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
- Merge Join Operator [MERGEJOIN_47] (rows=11 width=93)
- Conds:SEL_16._col0=SEL_19._col0(Inner),Output:["_col0"]
- <-Select Operator [SEL_19] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_44] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_17] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_16] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_43] (rows=10 width=93)
- predicate:key is not null
- TableScan [TS_14] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: llap
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0
+ native: false
+ projectedOutputColumns: [0]
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 5 Data size: 511 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 511 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 511 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 6
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0
+ native: false
+ projectedOutputColumns: [0]
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select src1.key, src1.cnt1, src2.cnt1 from
(
@@ -325,51 +641,117 @@ POSTHOOK: Input: default@tbl2
5 9 9
8 1 1
9 1 1
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join
(select a.key as key, a.value as value from tbl2 a where key < 6) subq2
on subq1.key = subq2.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join
(select a.key as key, a.value as value from tbl2 a where key < 6) subq2
on subq1.key = subq2.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=3 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
@@ -390,7 +772,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
20
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(
select * from
@@ -402,7 +784,7 @@ select count(*) from
join tbl2 b
on subq2.key = b.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(
select * from
@@ -414,37 +796,103 @@ select count(*) from
join tbl2 b
on subq2.key = b.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=1 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(
@@ -475,7 +923,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
20
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(
select * from
@@ -494,7 +942,7 @@ select count(*) from
) subq4
on subq2.key = subq4.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(
select * from
@@ -513,37 +961,103 @@ select count(*) from
) subq4
on subq2.key = subq4.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=1 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_3] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(
@@ -586,51 +1100,117 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl1
#### A masked pattern was here ####
20
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
join
(select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
on subq1.key = subq2.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
join
(select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
on subq1.key = subq2.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=3 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=3 width=93)
- predicate:(key < 8)
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=3 width=93)
- predicate:(key < 8)
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 8) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 8) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
@@ -651,58 +1231,183 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
20
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
join
(select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
on subq1.key = subq2.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
join
(select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
on subq1.key = subq2.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNotNull(col 2)(children: LongColAddLongScalar(col 0, val 1) -> 2:long) -> boolean
+ predicate: (key + 1) is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (key + 1) (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2]
+ selectExpressions: LongColAddLongScalar(col 0, val 1) -> 2:long
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNotNull(col 2)(children: LongColAddLongScalar(col 0, val 1) -> 2:long) -> boolean
+ predicate: (key + 1) is not null (type: boolean)
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (key + 1) (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2]
+ selectExpressions: LongColAddLongScalar(col 0, val 1) -> 2:long
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkLongOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, Not ACID UPDATE or DELETE IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 3 vectorized, llap
- File Output Operator [FS_29]
- Group By Operator [GBY_28] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_21] (rows=11 width=93)
- Conds:RS_24._col0=RS_27._col0(Inner)
- <-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_24]
- PartitionCols:_col0
- Select Operator [SEL_23] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_22] (rows=10 width=93)
- predicate:(key + 1) is not null
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Map 4 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_27]
- PartitionCols:_col0
- Select Operator [SEL_26] (rows=10 width=93)
- Output:["_col0"]
- Filter Operator [FIL_25] (rows=10 width=93)
- predicate:(key + 1) is not null
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
@@ -723,47 +1428,113 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
22
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join tbl2 a on subq1.key = a.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join tbl2 a on subq1.key = a.key
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=3 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
@@ -780,7 +1551,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
20
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join
@@ -790,7 +1561,7 @@ select count(*) from
(select a.key as key, a.value as value from tbl2 a where key < 6) subq3
on (subq1.key = subq3.key)
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
join
@@ -800,43 +1571,116 @@ select count(*) from
(select a.key as key, a.value as value from tbl2 a where key < 6) subq3
on (subq1.key = subq3.key)
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < 6) (type: boolean)
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0]
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_32]
- Group By Operator [GBY_31] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_15]
- Group By Operator [GBY_14] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_28] (rows=6 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner),SEL_2._col0=SEL_8._col0(Inner)
- <-Select Operator [SEL_5] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_26] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_8] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_27] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_6] (rows=10 width=93)
- default@tbl2,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=3 width=93)
- Output:["_col0"]
- Filter Operator [FIL_25] (rows=3 width=93)
- predicate:(key < 6)
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: select count(*) from
(select a.key as key, a.value as value from tbl1 a where key < 6) subq1
@@ -863,7 +1707,7 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
#### A masked pattern was here ####
56
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization expression
select count(*) from (
select subq2.key as key, subq2.value as value1, b.value as value2 from
(
@@ -876,7 +1720,7 @@ select count(*) from (
join tbl2 b
on subq2.key = b.key) a
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization expression
select count(*) from (
select subq2.key as key, subq2.value as value1, b.value as value2 from
(
@@ -889,37 +1733,103 @@ select count(*) from (
join tbl2 b
on subq2.key = b.key) a
POSTHOOK: type: QUERY
-Plan optimized by CBO.
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_22]
- Group By Operator [GBY_21] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=1 width=102)
- Conds:SEL_2._col0=SEL_5._col0(Inner)
- <-Select Operator [SEL_5] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_3] (rows=10 width=93)
- default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Select Operator [SEL_2] (rows=1 width=93)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=1 width=93)
- predicate:((key < 8) and (key < 6))
- TableScan [TS_0] (rows=10 width=93)
- default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key < 8) and (key < 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true,
<TRUNCATED>