You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/07/20 10:16:54 UTC
[28/36] hive git commit: HIVE-16369: Vectorization: Support PTF (Part
1: No Custom Window Framing -- Default Only) (Matt McCline,
reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/a0df0ace/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
index c2f5a29..805d5a2 100644
--- a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
@@ -164,16 +164,29 @@ STAGE PLANS:
dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
partitionColumnCount: 0
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
- vectorized: false
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, VALUE._col0:string, VALUE._col1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, bigint, bigint, double, double, bigint, bigint
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double)
outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
@@ -192,53 +205,72 @@ STAGE PLANS:
alias: row_number_window_0
name: row_number
window function: GenericUDAFRowNumberEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: rank_window_1
arguments: _col0
name: rank
window function: GenericUDAFRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: dense_rank_window_2
arguments: _col0
name: dense_rank
window function: GenericUDAFDenseRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: first_value_window_3
arguments: _col2
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
window function definition
alias: last_value_window_4
arguments: _col2
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
window function definition
alias: count_window_5
arguments: _col2
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
window function definition
alias: count_window_6
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isStar: true
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRowNumber, VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorDoubleFirstValue, VectorPTFEvaluatorDoubleLastValue, VectorPTFEvaluatorCount, VectorPTFEvaluatorCountStar]
+ functionInputExpressions: [null, col 0, col 0, col 2, col 2, col 2, null]
+ functionNames: [row_number, rank, dense_rank, first_value, last_value, count, count]
+ keyInputColumns: [0]
+ native: true
+ nonKeyInputColumns: [1, 2]
+ orderExpressions: [col 0]
+ outputColumns: [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]
+ outputTypes: [int, int, int, double, double, bigint, bigint, string, string, double]
+ streamingColumns: [3, 4, 5, 6]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), row_number_window_0 (type: int), rank_window_1 (type: int), dense_rank_window_2 (type: int), first_value_window_3 (type: double), last_value_window_4 (type: double), count_window_5 (type: bigint), count_window_6 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -318,24 +350,24 @@ Manufacturer#3 almond antique forest lavender goldenrod NULL 7 1 1 590.27 99.68
Manufacturer#3 almond antique chartreuse khaki white 99.68 8 1 1 590.27 99.68 7 8
PREHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name) as rn,
-rank() over(partition by p_mfgr order by p_name) as r,
-dense_rank() over(partition by p_mfgr order by p_name) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
-count(*) over(partition by p_mfgr order by p_name) as cs
+row_number() over(partition by p_mfgr range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr range between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name) as rn,
-rank() over(partition by p_mfgr order by p_name) as r,
-dense_rank() over(partition by p_mfgr order by p_name) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
-count(*) over(partition by p_mfgr order by p_name) as cs
+row_number() over(partition by p_mfgr range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr range between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
POSTHOOK: type: QUERY
Explain
@@ -364,18 +396,17 @@ STAGE PLANS:
native: true
projectedOutputColumns: [0, 1, 2]
Reduce Output Operator
- key expressions: p_mfgr (type: string), p_name (type: string)
- sort order: ++
+ key expressions: p_mfgr (type: string)
+ sort order: +
Map-reduce partition columns: p_mfgr (type: string)
Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumns: [0, 1]
+ className: VectorReduceSinkStringOperator
+ keyColumns: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumns: [0]
- valueColumns: [2]
+ valueColumns: [1, 2]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_retailprice (type: double)
+ value expressions: p_name (type: string), p_retailprice (type: double)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -396,11 +427,11 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: row_number only CURRENT ROW end frame is supported for RANGE
vectorized: false
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double)
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
PTF Operator
@@ -412,7 +443,7 @@ STAGE PLANS:
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col1 ASC NULLS FIRST
+ order by: _col0 ASC NULLS FIRST
partition by: _col0
raw input shape:
window functions:
@@ -420,45 +451,45 @@ STAGE PLANS:
alias: row_number_window_0
name: row_number
window function: GenericUDAFRowNumberEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: rank_window_1
- arguments: _col1
+ arguments: _col0
name: rank
window function: GenericUDAFRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: dense_rank_window_2
- arguments: _col1
+ arguments: _col0
name: dense_rank
window function: GenericUDAFDenseRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: first_value_window_3
arguments: _col2
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
window function definition
alias: last_value_window_4
arguments: _col2
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
window function definition
alias: count_window_5
arguments: _col2
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
window function definition
alias: count_window_6
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
isStar: true
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -480,90 +511,90 @@ STAGE PLANS:
ListSink
PREHOOK: query: select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name) as rn,
-rank() over(partition by p_mfgr order by p_name) as r,
-dense_rank() over(partition by p_mfgr order by p_name) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
-count(*) over(partition by p_mfgr order by p_name) as cs
+row_number() over(partition by p_mfgr range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr range between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@vector_ptf_part_simple_orc
#### A masked pattern was here ####
POSTHOOK: query: select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name) as rn,
-rank() over(partition by p_mfgr order by p_name) as r,
-dense_rank() over(partition by p_mfgr order by p_name) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
-count(*) over(partition by p_mfgr order by p_name) as cs
+row_number() over(partition by p_mfgr range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr range between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_ptf_part_simple_orc
#### A masked pattern was here ####
p_mfgr p_name p_retailprice rn r dr fv lv c cs
-Manufacturer#1 almond antique burnished rose metallic 1173.15 1 1 1 1173.15 1173.15 2 2
-Manufacturer#1 almond antique burnished rose metallic 1173.15 2 1 1 1173.15 1173.15 2 2
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 3 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 5 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 6 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 7 3 1173.15 1602.59 7 7
-Manufacturer#1 almond aquamarine burnished black steel 1414.42 8 8 4 1173.15 1414.42 8 8
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 9 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle NULL 10 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 11 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 9 5 1173.15 1632.66 11 12
-Manufacturer#2 almond antique violet chocolate turquoise 1690.68 1 1 1 1690.68 1690.68 1 1
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 2 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 4 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond aquamarine midnight light salmon 2031.98 5 5 3 1690.68 2031.98 5 5
-Manufacturer#2 almond aquamarine rose maroon antique 900.66 6 6 4 1690.68 1698.66 7 7
-Manufacturer#2 almond aquamarine rose maroon antique 1698.66 7 6 4 1690.68 1698.66 7 7
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 8 8 5 1690.68 1000.6 8 8
-Manufacturer#3 almond antique chartreuse khaki white 99.68 1 1 1 99.68 99.68 1 1
-Manufacturer#3 almond antique forest lavender goldenrod 590.27 2 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod NULL 3 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod 1190.27 4 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod 1190.27 5 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique metallic orange dim 55.39 6 6 3 99.68 55.39 5 6
-Manufacturer#3 almond antique misty red olive 1922.98 7 7 4 99.68 1922.98 6 7
-Manufacturer#3 almond antique olive coral navajo 1337.29 8 8 5 99.68 1337.29 7 8
-Manufacturer#4 almond antique gainsboro frosted violet NULL 1 1 1 NULL NULL 0 1
-Manufacturer#4 almond antique violet mint lemon 1375.42 2 2 2 NULL 1375.42 1 2
-Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 3 3 NULL 1206.26 2 4
-Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 4 3 3 NULL 1206.26 2 4
-Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 5 4 NULL 1844.92 3 5
-Manufacturer#4 almond azure aquamarine papaya violet 1290.35 6 6 5 NULL 1290.35 4 6
-Manufacturer#5 almond antique blue firebrick mint 1789.69 1 1 1 1789.69 1789.69 1 1
-Manufacturer#5 almond antique medium spring khaki 1611.66 2 2 2 1789.69 1611.66 3 3
-Manufacturer#5 almond antique medium spring khaki 1611.66 3 2 2 1789.69 1611.66 3 3
-Manufacturer#5 almond antique sky peru orange 1788.73 4 4 3 1789.69 1788.73 4 4
-Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 5 5 4 1789.69 1018.1 5 5
-Manufacturer#5 almond azure blanched chiffon midnight 1464.48 6 6 5 1789.69 1464.48 6 6
+Manufacturer#4 almond azure aquamarine papaya violet 1290.35 1 1 1 1290.35 1206.26 4 6
+Manufacturer#4 almond antique violet mint lemon 1375.42 2 1 1 1290.35 1206.26 4 6
+Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 1 1 1290.35 1206.26 4 6
+Manufacturer#4 almond antique gainsboro frosted violet NULL 4 1 1 1290.35 1206.26 4 6
+Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 1 1 1290.35 1206.26 4 6
+Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 6 1 1 1290.35 1206.26 4 6
+Manufacturer#5 almond azure blanched chiffon midnight 1464.48 1 1 1 1464.48 1788.73 6 6
+Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 2 1 1 1464.48 1788.73 6 6
+Manufacturer#5 almond antique medium spring khaki 1611.66 3 1 1 1464.48 1788.73 6 6
+Manufacturer#5 almond antique blue firebrick mint 1789.69 4 1 1 1464.48 1788.73 6 6
+Manufacturer#5 almond antique medium spring khaki 1611.66 5 1 1 1464.48 1788.73 6 6
+Manufacturer#5 almond antique sky peru orange 1788.73 6 1 1 1464.48 1788.73 6 6
+Manufacturer#2 almond aquamarine rose maroon antique 900.66 1 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond aquamarine rose maroon antique 1698.66 2 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond antique violet chocolate turquoise 1690.68 4 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 5 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 6 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond aquamarine midnight light salmon 2031.98 7 1 1 900.66 1800.7 8 8
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 8 1 1 900.66 1800.7 8 8
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 1 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 2 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 3 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond aquamarine burnished black steel 1414.42 5 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique burnished rose metallic 1173.15 6 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 8 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique burnished rose metallic 1173.15 9 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 10 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle NULL 11 1 1 1753.76 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 1 1 1753.76 1632.66 11 12
+Manufacturer#3 almond antique forest lavender goldenrod 590.27 1 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique metallic orange dim 55.39 2 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 3 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique olive coral navajo 1337.29 4 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique misty red olive 1922.98 5 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 6 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique forest lavender goldenrod NULL 7 1 1 590.27 99.68 7 8
+Manufacturer#3 almond antique chartreuse khaki white 99.68 8 1 1 590.27 99.68 7 8
PREHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
-rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
-dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
-count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+row_number() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr rows between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
-rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
-dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
-count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+row_number() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr rows between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
POSTHOOK: type: QUERY
Explain
@@ -592,18 +623,17 @@ STAGE PLANS:
native: true
projectedOutputColumns: [0, 1, 2]
Reduce Output Operator
- key expressions: p_mfgr (type: string), p_name (type: string)
- sort order: ++
+ key expressions: p_mfgr (type: string)
+ sort order: +
Map-reduce partition columns: p_mfgr (type: string)
Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumns: [0, 1]
+ className: VectorReduceSinkStringOperator
+ keyColumns: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumns: [0]
- valueColumns: [2]
+ valueColumns: [1, 2]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_retailprice (type: double)
+ value expressions: p_name (type: string), p_retailprice (type: double)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -624,11 +654,11 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: first_value UNBOUNDED end frame is not supported for ROWS window type
vectorized: false
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double)
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
PTF Operator
@@ -640,7 +670,7 @@ STAGE PLANS:
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col1 ASC NULLS FIRST
+ order by: _col0 ASC NULLS FIRST
partition by: _col0
raw input shape:
window functions:
@@ -648,45 +678,45 @@ STAGE PLANS:
alias: row_number_window_0
name: row_number
window function: GenericUDAFRowNumberEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: rank_window_1
- arguments: _col1
+ arguments: _col0
name: rank
window function: GenericUDAFRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: dense_rank_window_2
- arguments: _col1
+ arguments: _col0
name: dense_rank
window function: GenericUDAFDenseRankEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
window function definition
alias: first_value_window_3
arguments: _col2
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: ROWS PRECEDING(MAX)~CURRENT
window function definition
alias: last_value_window_4
arguments: _col2
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: ROWS PRECEDING(MAX)~CURRENT
window function definition
alias: count_window_5
arguments: _col2
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: ROWS PRECEDING(MAX)~CURRENT
window function definition
alias: count_window_6
name: count
window function: GenericUDAFCountEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: ROWS PRECEDING(MAX)~CURRENT
isStar: true
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -708,84 +738,1978 @@ STAGE PLANS:
ListSink
PREHOOK: query: select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
-rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
-dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
-count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+row_number() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr rows between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@vector_ptf_part_simple_orc
#### A masked pattern was here ####
POSTHOOK: query: select p_mfgr,p_name, p_retailprice,
-row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
-rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
-dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
-first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
-last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
-count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
-count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+row_number() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr rows between unbounded preceding and current row) as cs
from vector_ptf_part_simple_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_ptf_part_simple_orc
#### A masked pattern was here ####
p_mfgr p_name p_retailprice rn r dr fv lv c cs
-Manufacturer#1 almond antique burnished rose metallic 1173.15 1 1 1 1173.15 1173.15 2 2
-Manufacturer#1 almond antique burnished rose metallic 1173.15 2 1 1 1173.15 1173.15 2 2
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 3 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 5 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 6 3 2 1173.15 1753.76 6 6
-Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 7 3 1173.15 1602.59 7 7
-Manufacturer#1 almond aquamarine burnished black steel 1414.42 8 8 4 1173.15 1414.42 8 8
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 9 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle NULL 10 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 11 9 5 1173.15 1632.66 11 12
-Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 9 5 1173.15 1632.66 11 12
-Manufacturer#2 almond antique violet chocolate turquoise 1690.68 1 1 1 1690.68 1690.68 1 1
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 2 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond antique violet turquoise frosted 1800.7 4 2 2 1690.68 1800.7 4 4
-Manufacturer#2 almond aquamarine midnight light salmon 2031.98 5 5 3 1690.68 2031.98 5 5
-Manufacturer#2 almond aquamarine rose maroon antique 900.66 6 6 4 1690.68 1698.66 7 7
-Manufacturer#2 almond aquamarine rose maroon antique 1698.66 7 6 4 1690.68 1698.66 7 7
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 8 8 5 1690.68 1000.6 8 8
-Manufacturer#3 almond antique chartreuse khaki white 99.68 1 1 1 99.68 99.68 1 1
-Manufacturer#3 almond antique forest lavender goldenrod 590.27 2 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod NULL 3 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod 1190.27 4 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique forest lavender goldenrod 1190.27 5 2 2 99.68 1190.27 4 5
-Manufacturer#3 almond antique metallic orange dim 55.39 6 6 3 99.68 55.39 5 6
-Manufacturer#3 almond antique misty red olive 1922.98 7 7 4 99.68 1922.98 6 7
-Manufacturer#3 almond antique olive coral navajo 1337.29 8 8 5 99.68 1337.29 7 8
-Manufacturer#4 almond antique gainsboro frosted violet NULL 1 1 1 NULL NULL 0 1
-Manufacturer#4 almond antique violet mint lemon 1375.42 2 2 2 NULL 1375.42 1 2
-Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 3 3 NULL 1206.26 2 4
-Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 4 3 3 NULL 1206.26 2 4
-Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 5 4 NULL 1844.92 3 5
-Manufacturer#4 almond azure aquamarine papaya violet 1290.35 6 6 5 NULL 1290.35 4 6
-Manufacturer#5 almond antique blue firebrick mint 1789.69 1 1 1 1789.69 1789.69 1 1
-Manufacturer#5 almond antique medium spring khaki 1611.66 2 2 2 1789.69 1611.66 3 3
-Manufacturer#5 almond antique medium spring khaki 1611.66 3 2 2 1789.69 1611.66 3 3
-Manufacturer#5 almond antique sky peru orange 1788.73 4 4 3 1789.69 1788.73 4 4
-Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 5 5 4 1789.69 1018.1 5 5
-Manufacturer#5 almond azure blanched chiffon midnight 1464.48 6 6 5 1789.69 1464.48 6 6
+Manufacturer#4 almond azure aquamarine papaya violet 1290.35 1 1 1 1290.35 1290.35 1 1
+Manufacturer#4 almond antique violet mint lemon 1375.42 2 1 1 1290.35 1375.42 2 2
+Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 1 1 1290.35 NULL 2 3
+Manufacturer#4 almond antique gainsboro frosted violet NULL 4 1 1 1290.35 NULL 2 4
+Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 1 1 1290.35 1844.92 3 5
+Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 6 1 1 1290.35 1206.26 4 6
+Manufacturer#5 almond azure blanched chiffon midnight 1464.48 1 1 1 1464.48 1464.48 1 1
+Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 2 1 1 1464.48 1018.1 2 2
+Manufacturer#5 almond antique medium spring khaki 1611.66 3 1 1 1464.48 1611.66 3 3
+Manufacturer#5 almond antique blue firebrick mint 1789.69 4 1 1 1464.48 1789.69 4 4
+Manufacturer#5 almond antique medium spring khaki 1611.66 5 1 1 1464.48 1611.66 5 5
+Manufacturer#5 almond antique sky peru orange 1788.73 6 1 1 1464.48 1788.73 6 6
+Manufacturer#2 almond aquamarine rose maroon antique 900.66 1 1 1 900.66 900.66 1 1
+Manufacturer#2 almond aquamarine rose maroon antique 1698.66 2 1 1 900.66 1698.66 2 2
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 1 1 900.66 1800.7 3 3
+Manufacturer#2 almond antique violet chocolate turquoise 1690.68 4 1 1 900.66 1690.68 4 4
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 5 1 1 900.66 1800.7 5 5
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 6 1 1 900.66 1000.6 6 6
+Manufacturer#2 almond aquamarine midnight light salmon 2031.98 7 1 1 900.66 2031.98 7 7
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 8 1 1 900.66 1800.7 8 8
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 1 1 1 1753.76 1753.76 1 1
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 2 1 1 1753.76 1632.66 2 2
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 3 1 1 1753.76 1632.66 3 3
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 1 1 1753.76 1753.76 4 4
+Manufacturer#1 almond aquamarine burnished black steel 1414.42 5 1 1 1753.76 1414.42 5 5
+Manufacturer#1 almond antique burnished rose metallic 1173.15 6 1 1 1753.76 1173.15 6 6
+Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 1 1 1753.76 1602.59 7 7
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 8 1 1 1753.76 1753.76 8 8
+Manufacturer#1 almond antique burnished rose metallic 1173.15 9 1 1 1753.76 1173.15 9 9
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 10 1 1 1753.76 1753.76 10 10
+Manufacturer#1 almond aquamarine pink moccasin thistle NULL 11 1 1 1753.76 NULL 10 11
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 1 1 1753.76 1632.66 11 12
+Manufacturer#3 almond antique forest lavender goldenrod 590.27 1 1 1 590.27 590.27 1 1
+Manufacturer#3 almond antique metallic orange dim 55.39 2 1 1 590.27 55.39 2 2
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 3 1 1 590.27 1190.27 3 3
+Manufacturer#3 almond antique olive coral navajo 1337.29 4 1 1 590.27 1337.29 4 4
+Manufacturer#3 almond antique misty red olive 1922.98 5 1 1 590.27 1922.98 5 5
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 6 1 1 590.27 1190.27 6 6
+Manufacturer#3 almond antique forest lavender goldenrod NULL 7 1 1 590.27 NULL 6 7
+Manufacturer#3 almond antique chartreuse khaki white 99.68 8 1 1 590.27 99.68 7 8
PREHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-sum(p_retailprice) over(partition by p_mfgr) as s,
-min(p_retailprice) over(partition by p_mfgr) as mi,
-max(p_retailprice) over(partition by p_mfgr) as ma,
-avg(p_retailprice) over(partition by p_mfgr) as av
+row_number() over(partition by p_mfgr order by p_name) as rn,
+rank() over(partition by p_mfgr order by p_name) as r,
+dense_rank() over(partition by p_mfgr order by p_name) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
+count(*) over(partition by p_mfgr order by p_name) as cs
from vector_ptf_part_simple_orc
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
select p_mfgr,p_name, p_retailprice,
-sum(p_retailprice) over(partition by p_mfgr) as s,
-min(p_retailprice) over(partition by p_mfgr) as mi,
-max(p_retailprice) over(partition by p_mfgr) as ma,
-avg(p_retailprice) over(partition by p_mfgr) as av
+row_number() over(partition by p_mfgr order by p_name) as rn,
+rank() over(partition by p_mfgr order by p_name) as r,
+dense_rank() over(partition by p_mfgr order by p_name) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
+count(*) over(partition by p_mfgr order by p_name) as cs
+from vector_ptf_part_simple_orc
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_part_simple_orc
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2]
+ Reduce Output Operator
+ key expressions: p_mfgr (type: string), p_name (type: string)
+ sort order: ++
+ Map-reduce partition columns: p_mfgr (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0]
+ valueColumns: [2]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ value expressions: p_retailprice (type: double)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
+ partitionColumnCount: 0
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:string, KEY.reducesinkkey1:string, VALUE._col0:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, bigint, bigint, double, double, bigint, bigint
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: string, _col2: double
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: rank_window_1
+ arguments: _col1
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_2
+ arguments: _col1
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: first_value_window_3
+ arguments: _col2
+ name: first_value
+ window function: GenericUDAFFirstValueEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: last_value_window_4
+ arguments: _col2
+ name: last_value
+ window function: GenericUDAFLastValueEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_5
+ arguments: _col2
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_6
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ isStar: true
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRowNumber, VectorPTFEvaluatorRank, VectorPTFEvaluatorDenseRank, VectorPTFEvaluatorDoubleFirstValue, VectorPTFEvaluatorDoubleLastValue, VectorPTFEvaluatorCount, VectorPTFEvaluatorCountStar]
+ functionInputExpressions: [null, col 1, col 1, col 2, col 2, col 2, null]
+ functionNames: [row_number, rank, dense_rank, first_value, last_value, count, count]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1]
+ outputColumns: [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]
+ outputTypes: [int, int, int, double, double, bigint, bigint, string, string, double]
+ partitionExpressions: [col 0]
+ streamingColumns: [3, 4, 5, 6]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), row_number_window_0 (type: int), rank_window_1 (type: int), dense_rank_window_2 (type: int), first_value_window_3 (type: double), last_value_window_4 (type: double), count_window_5 (type: bigint), count_window_6 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name) as rn,
+rank() over(partition by p_mfgr order by p_name) as r,
+dense_rank() over(partition by p_mfgr order by p_name) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
+count(*) over(partition by p_mfgr order by p_name) as cs
+from vector_ptf_part_simple_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_part_simple_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name) as rn,
+rank() over(partition by p_mfgr order by p_name) as r,
+dense_rank() over(partition by p_mfgr order by p_name) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name) as c,
+count(*) over(partition by p_mfgr order by p_name) as cs
+from vector_ptf_part_simple_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_part_simple_orc
+#### A masked pattern was here ####
+p_mfgr p_name p_retailprice rn r dr fv lv c cs
+Manufacturer#1 almond antique burnished rose metallic 1173.15 1 1 1 1173.15 1173.15 2 2
+Manufacturer#1 almond antique burnished rose metallic 1173.15 2 1 1 1173.15 1173.15 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 3 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 5 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 6 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 7 3 1173.15 1602.59 7 7
+Manufacturer#1 almond aquamarine burnished black steel 1414.42 8 8 4 1173.15 1414.42 8 8
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 9 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle NULL 10 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 11 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 9 5 1173.15 1632.66 11 12
+Manufacturer#2 almond antique violet chocolate turquoise 1690.68 1 1 1 1690.68 1690.68 1 1
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 2 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 4 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond aquamarine midnight light salmon 2031.98 5 5 3 1690.68 2031.98 5 5
+Manufacturer#2 almond aquamarine rose maroon antique 900.66 6 6 4 1690.68 1698.66 7 7
+Manufacturer#2 almond aquamarine rose maroon antique 1698.66 7 6 4 1690.68 1698.66 7 7
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 8 8 5 1690.68 1000.6 8 8
+Manufacturer#3 almond antique chartreuse khaki white 99.68 1 1 1 99.68 99.68 1 1
+Manufacturer#3 almond antique forest lavender goldenrod 590.27 2 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod NULL 3 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 4 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 5 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique metallic orange dim 55.39 6 6 3 99.68 55.39 5 6
+Manufacturer#3 almond antique misty red olive 1922.98 7 7 4 99.68 1922.98 6 7
+Manufacturer#3 almond antique olive coral navajo 1337.29 8 8 5 99.68 1337.29 7 8
+Manufacturer#4 almond antique gainsboro frosted violet NULL 1 1 1 NULL NULL 0 1
+Manufacturer#4 almond antique violet mint lemon 1375.42 2 2 2 NULL 1375.42 1 2
+Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 3 3 NULL 1206.26 2 4
+Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 4 3 3 NULL 1206.26 2 4
+Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 5 4 NULL 1844.92 3 5
+Manufacturer#4 almond azure aquamarine papaya violet 1290.35 6 6 5 NULL 1290.35 4 6
+Manufacturer#5 almond antique blue firebrick mint 1789.69 1 1 1 1789.69 1789.69 1 1
+Manufacturer#5 almond antique medium spring khaki 1611.66 2 2 2 1789.69 1611.66 3 3
+Manufacturer#5 almond antique medium spring khaki 1611.66 3 2 2 1789.69 1611.66 3 3
+Manufacturer#5 almond antique sky peru orange 1788.73 4 4 3 1789.69 1788.73 4 4
+Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 5 5 4 1789.69 1018.1 5 5
+Manufacturer#5 almond azure blanched chiffon midnight 1464.48 6 6 5 1789.69 1464.48 6 6
+PREHOOK: query: explain vectorization detail
+select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization detail
+select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_part_simple_orc
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2]
+ Reduce Output Operator
+ key expressions: p_mfgr (type: string), p_name (type: string)
+ sort order: ++
+ Map-reduce partition columns: p_mfgr (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0]
+ valueColumns: [2]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ value expressions: p_retailprice (type: double)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
+ partitionColumnCount: 0
+ Reducer 2
+ Execution mode: llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ notVectorizedReason: PTF operator: row_number only CURRENT ROW end frame is supported for RANGE
+ vectorized: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: string, _col2: double
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: rank_window_1
+ arguments: _col1
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_2
+ arguments: _col1
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: RANGE PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: first_value_window_3
+ arguments: _col2
+ name: first_value
+ window function: GenericUDAFFirstValueEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: last_value_window_4
+ arguments: _col2
+ name: last_value
+ window function: GenericUDAFLastValueEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_5
+ arguments: _col2
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_6
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ isStar: true
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), row_number_window_0 (type: int), rank_window_1 (type: int), dense_rank_window_2 (type: int), first_value_window_3 (type: double), last_value_window_4 (type: double), count_window_5 (type: bigint), count_window_6 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_part_simple_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name range between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name range between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vector_ptf_part_simple_orc
+#### A masked pattern was here ####
+p_mfgr p_name p_retailprice rn r dr fv lv c cs
+Manufacturer#1 almond antique burnished rose metallic 1173.15 1 1 1 1173.15 1173.15 2 2
+Manufacturer#1 almond antique burnished rose metallic 1173.15 2 1 1 1173.15 1173.15 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 3 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 4 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 5 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1753.76 6 3 2 1173.15 1753.76 6 6
+Manufacturer#1 almond antique salmon chartreuse burlywood 1602.59 7 7 3 1173.15 1602.59 7 7
+Manufacturer#1 almond aquamarine burnished black steel 1414.42 8 8 4 1173.15 1414.42 8 8
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 9 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle NULL 10 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 11 9 5 1173.15 1632.66 11 12
+Manufacturer#1 almond aquamarine pink moccasin thistle 1632.66 12 9 5 1173.15 1632.66 11 12
+Manufacturer#2 almond antique violet chocolate turquoise 1690.68 1 1 1 1690.68 1690.68 1 1
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 2 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 3 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond antique violet turquoise frosted 1800.7 4 2 2 1690.68 1800.7 4 4
+Manufacturer#2 almond aquamarine midnight light salmon 2031.98 5 5 3 1690.68 2031.98 5 5
+Manufacturer#2 almond aquamarine rose maroon antique 900.66 6 6 4 1690.68 1698.66 7 7
+Manufacturer#2 almond aquamarine rose maroon antique 1698.66 7 6 4 1690.68 1698.66 7 7
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1000.6 8 8 5 1690.68 1000.6 8 8
+Manufacturer#3 almond antique chartreuse khaki white 99.68 1 1 1 99.68 99.68 1 1
+Manufacturer#3 almond antique forest lavender goldenrod 590.27 2 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod NULL 3 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 4 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique forest lavender goldenrod 1190.27 5 2 2 99.68 1190.27 4 5
+Manufacturer#3 almond antique metallic orange dim 55.39 6 6 3 99.68 55.39 5 6
+Manufacturer#3 almond antique misty red olive 1922.98 7 7 4 99.68 1922.98 6 7
+Manufacturer#3 almond antique olive coral navajo 1337.29 8 8 5 99.68 1337.29 7 8
+Manufacturer#4 almond antique gainsboro frosted violet NULL 1 1 1 NULL NULL 0 1
+Manufacturer#4 almond antique violet mint lemon 1375.42 2 2 2 NULL 1375.42 1 2
+Manufacturer#4 almond aquamarine floral ivory bisque NULL 3 3 3 NULL 1206.26 2 4
+Manufacturer#4 almond aquamarine floral ivory bisque 1206.26 4 3 3 NULL 1206.26 2 4
+Manufacturer#4 almond aquamarine yellow dodger mint 1844.92 5 5 4 NULL 1844.92 3 5
+Manufacturer#4 almond azure aquamarine papaya violet 1290.35 6 6 5 NULL 1290.35 4 6
+Manufacturer#5 almond antique blue firebrick mint 1789.69 1 1 1 1789.69 1789.69 1 1
+Manufacturer#5 almond antique medium spring khaki 1611.66 2 2 2 1789.69 1611.66 3 3
+Manufacturer#5 almond antique medium spring khaki 1611.66 3 2 2 1789.69 1611.66 3 3
+Manufacturer#5 almond antique sky peru orange 1788.73 4 4 3 1789.69 1788.73 4 4
+Manufacturer#5 almond aquamarine dodger light gainsboro 1018.1 5 5 4 1789.69 1018.1 5 5
+Manufacturer#5 almond azure blanched chiffon midnight 1464.48 6 6 5 1789.69 1464.48 6 6
+PREHOOK: query: explain vectorization detail
+select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization detail
+select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vector_ptf_part_simple_orc
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2]
+ Reduce Output Operator
+ key expressions: p_mfgr (type: string), p_name (type: string)
+ sort order: ++
+ Map-reduce partition columns: p_mfgr (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0]
+ valueColumns: [2]
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ value expressions: p_retailprice (type: double)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
+ partitionColumnCount: 0
+ Reducer 2
+ Execution mode: llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ notVectorizedReason: PTF operator: first_value UNBOUNDED end frame is not supported for ROWS window type
+ vectorized: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: string, _col2: double
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: rank_window_1
+ arguments: _col1
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: dense_rank_window_2
+ arguments: _col1
+ name: dense_rank
+ window function: GenericUDAFDenseRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ window function definition
+ alias: first_value_window_3
+ arguments: _col2
+ name: first_value
+ window function: GenericUDAFFirstValueEvaluator
+ window frame: ROWS PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: last_value_window_4
+ arguments: _col2
+ name: last_value
+ window function: GenericUDAFLastValueEvaluator
+ window frame: ROWS PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_5
+ arguments: _col2
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: ROWS PRECEDING(MAX)~CURRENT
+ window function definition
+ alias: count_window_6
+ name: count
+ window function: GenericUDAFCountEvaluator
+ window frame: ROWS PRECEDING(MAX)~CURRENT
+ isStar: true
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), row_number_window_0 (type: int), rank_window_1 (type: int), dense_rank_window_2 (type: int), first_value_window_3 (type: double), last_value_window_4 (type: double), count_window_5 (type: bigint), count_window_6 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as dr,
+first_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as fv,
+last_value(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as lv,
+count(p_retailprice) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as c,
+count(*) over(partition by p_mfgr order by p_name rows between unbounded preceding and current row) as cs
+from vector_ptf_part_simple_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vector_ptf_part_simple_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_retailprice,
+row_number() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as rn,
+rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as r,
+dense_rank() over(partition by p_mfgr order by p_name rows between unbounded preceding and unbounded following) as dr,
<TRUNCATED>