You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/07/20 10:16:48 UTC
[22/36] hive git commit: HIVE-16369: Vectorization: Support PTF (Part
1: No Custom Window Framing -- Default Only) (Matt McCline,
reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/a0df0ace/ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out
index 9e62256..6d66a55 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing_navfn.q.out
@@ -111,12 +111,21 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: bigint
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
- vectorized: false
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 1
+ dataColumns: KEY.reducesinkkey0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, bigint
Reduce Operator Tree:
PTF Operator
Function definitions:
@@ -135,15 +144,34 @@ STAGE PLANS:
alias: row_number_window_0
name: row_number
window function: GenericUDAFRowNumberEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRowNumber]
+ functionInputExpressions: [null]
+ functionNames: [row_number]
+ keyInputColumns: []
+ native: true
+ nonKeyInputColumns: []
+ orderExpressions: [ConstantVectorExpression(val 0) -> 2:long]
+ outputColumns: [1]
+ outputTypes: [int]
+ streamingColumns: [1]
Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: row_number_window_0 (type: int)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [1]
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -227,16 +255,29 @@ STAGE PLANS:
dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(4,2), bin:binary
partitionColumnCount: 0
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
- vectorized: false
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:double, KEY.reducesinkkey1:decimal(4,2), VALUE._col6:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col6 (type: string), KEY.reducesinkkey1 (type: decimal(4,2))
outputColumnNames: _col5, _col7, _col9
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 2, 1]
Statistics: Num rows: 4625 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
@@ -255,18 +296,41 @@ STAGE PLANS:
alias: row_number_window_0
name: row_number
window function: GenericUDAFRowNumberEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorRowNumber]
+ functionInputExpressions: [null]
+ functionNames: [row_number]
+ keyInputColumns: [0, 1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1]
+ outputColumns: [3, 0, 2, 1]
+ outputTypes: [int, double, string, decimal(4,2)]
+ partitionExpressions: [col 0]
+ streamingColumns: [3]
Statistics: Num rows: 4625 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), row_number_window_0 (type: int)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3]
Statistics: Num rows: 4625 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 100 Data size: 22000 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 100 Data size: 22000 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -452,7 +516,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: lead not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -477,7 +541,7 @@ STAGE PLANS:
arguments: _col7
name: lead
window function: GenericUDAFLeadEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 4799 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -673,7 +737,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum]
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -698,7 +762,7 @@ STAGE PLANS:
arguments: _col9
name: lag
window function: GenericUDAFLagEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(MAX)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 4710 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -891,16 +955,29 @@ STAGE PLANS:
dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(4,2), bin:binary
partitionColumnCount: 0
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
- vectorized: false
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY.reducesinkkey0:double, KEY.reducesinkkey1:float, VALUE._col0:tinyint, VALUE._col5:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey1 (type: float), KEY.reducesinkkey0 (type: double), VALUE._col5 (type: string)
outputColumnNames: _col0, _col4, _col5, _col7
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 1, 0, 3]
Statistics: Num rows: 8771 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
@@ -920,17 +997,40 @@ STAGE PLANS:
arguments: _col0
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorLongLastValue]
+ functionInputExpressions: [col 2]
+ functionNames: [last_value]
+ keyInputColumns: [1, 0]
+ native: true
+ nonKeyInputColumns: [2, 3]
+ orderExpressions: [col 1]
+ outputColumns: [4, 2, 1, 0, 3]
+ outputTypes: [tinyint, tinyint, float, double, string]
+ partitionExpressions: [col 0]
+ streamingColumns: []
Statistics: Num rows: 8771 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), last_value_window_0 (type: tinyint)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [3, 4]
Statistics: Num rows: 8771 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 100 Data size: 11600 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 100 Data size: 11600 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -1115,7 +1215,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: string data type not supported in argument expression of aggregation function first_value
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -1140,7 +1240,7 @@ STAGE PLANS:
arguments: _col7
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
Statistics: Num rows: 9784 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col7 (type: string), first_value_window_0 (type: string)
@@ -1343,16 +1443,29 @@ STAGE PLANS:
partitionColumnCount: 0
scratchColumnTypeNames: bigint, bigint
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
- vectorized: false
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:string, VALUE._col2:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, bigint, bigint
Reduce Operator Tree:
Select Operator
expressions: VALUE._col2 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col2, _col7
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 1]
Statistics: Num rows: 4710 Data size: 508717 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
@@ -1372,14 +1485,35 @@ STAGE PLANS:
arguments: _col2
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~CURRENT
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorLongLastValue]
+ functionInputExpressions: [col 2]
+ functionNames: [last_value]
+ keyInputColumns: [1]
+ native: true
+ nonKeyInputColumns: [2]
+ orderExpressions: [col 1]
+ outputColumns: [3, 2, 1]
+ outputTypes: [int, int, string]
+ partitionExpressions: [ConstantVectorExpression(val 10) -> 4:long]
+ streamingColumns: []
Statistics: Num rows: 4710 Data size: 508717 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 10 (type: tinyint), _col7 (type: string), _col2 (type: int), last_value_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [5, 1, 2, 3]
+ selectExpressions: ConstantVectorExpression(val 10) -> 5:long
Statistics: Num rows: 4710 Data size: 508717 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 4710 Data size: 508717 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -1504,7 +1638,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: first_value only UNBOUNDED start frame is supported
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -1529,25 +1663,25 @@ STAGE PLANS:
arguments: _col1
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: first_value_window_1
arguments: _col1, true
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: first_value_window_2
arguments: _col1
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
window function definition
alias: first_value_window_3
arguments: _col1, true
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
Statistics: Num rows: 15 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int), first_value_window_0 (type: int), first_value_window_1 (type: int), first_value_window_2 (type: int), first_value_window_3 (type: int)
@@ -1674,7 +1808,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: first_value only UNBOUNDED start frame is supported
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -1699,25 +1833,25 @@ STAGE PLANS:
arguments: _col1
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: first_value_window_1
arguments: _col1, true
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: first_value_window_2
arguments: _col1
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
window function definition
alias: first_value_window_3
arguments: _col1, true
name: first_value
window function: GenericUDAFFirstValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
Statistics: Num rows: 15 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int), first_value_window_0 (type: int), first_value_window_1 (type: int), first_value_window_2 (type: int), first_value_window_3 (type: int)
@@ -1844,7 +1978,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: last_value only UNBOUNDED start frame is supported
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -1869,25 +2003,25 @@ STAGE PLANS:
arguments: _col1
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: last_value_window_1
arguments: _col1, true
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: last_value_window_2
arguments: _col1
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
window function definition
alias: last_value_window_3
arguments: _col1, true
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
Statistics: Num rows: 15 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int), last_value_window_0 (type: int), last_value_window_1 (type: int), last_value_window_2 (type: int), last_value_window_3 (type: int)
@@ -2014,7 +2148,7 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- notVectorizedReason: PTF Operator (PTF) not supported
+ notVectorizedReason: PTF operator: last_value only UNBOUNDED start frame is supported
vectorized: false
Reduce Operator Tree:
Select Operator
@@ -2039,25 +2173,25 @@ STAGE PLANS:
arguments: _col1
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: last_value_window_1
arguments: _col1, true
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(1)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(1)~FOLLOWING(1)
window function definition
alias: last_value_window_2
arguments: _col1
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
window function definition
alias: last_value_window_3
arguments: _col1, true
name: last_value
window function: GenericUDAFLastValueEvaluator
- window frame: PRECEDING(MAX)~FOLLOWING(1)
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(1)
Statistics: Num rows: 15 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int), last_value_window_0 (type: int), last_value_window_1 (type: int), last_value_window_2 (type: int), last_value_window_3 (type: int)