You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/10/29 20:40:01 UTC
[22/51] [partial] hive git commit: HIVE-17433: Vectorization: Support
Decimal64 in Hive Query Engine (Matt McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
index ca3e669..72cd1d3 100644
--- a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
@@ -152,12 +152,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:char(10))
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -166,7 +165,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -196,7 +195,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -208,12 +208,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:char(10))
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -222,7 +221,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: char(10))
@@ -239,7 +238,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -249,7 +249,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -260,7 +259,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ projectedOutputColumnNums: [0, 1, 2, 3]
Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -320,12 +319,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:char(10))
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -334,7 +332,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: char(20))
@@ -351,7 +349,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -363,12 +362,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:char(20))
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -377,7 +375,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 324 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -407,7 +405,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -417,7 +416,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -428,7 +426,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ projectedOutputColumnNums: [0, 1, 2, 3]
Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -490,12 +488,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:char(10))
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -504,7 +501,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: UDFToString(_col1) (type: string)
@@ -512,7 +509,7 @@ STAGE PLANS:
Map-reduce partition columns: UDFToString(_col1) (type: string)
Reduce Sink Vectorization:
className: VectorReduceSinkStringOperator
- keyExpressions: CastStringGroupToString(col 1) -> 2:String
+ keyExpressions: CastStringGroupToString(col 1:char(10)) -> 3:string
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
@@ -522,7 +519,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -534,12 +532,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 1) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 1:string)
predicate: c2 is not null (type: boolean)
Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -548,7 +545,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -578,7 +575,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -588,7 +586,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -599,7 +596,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ projectedOutputColumnNums: [0, 1, 2, 3]
Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_char_simple.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_char_simple.q.out b/ql/src/test/results/clientpositive/llap/vector_char_simple.q.out
index 47c709f..696359b 100644
--- a/ql/src/test/results/clientpositive/llap/vector_char_simple.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_char_simple.q.out
@@ -75,7 +75,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -85,7 +86,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -161,7 +161,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -171,7 +172,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -259,7 +259,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -269,7 +270,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -283,7 +283,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- selectExpressions: CastLongToChar(col 0, maxLength 12) -> 1:Char
+ selectExpressions: CastLongToChar(col 0:int, maxLength 12) -> 1:char(12)
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out
index eb8ec44..0ebcb84 100644
--- a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out
@@ -28,16 +28,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNull(col 5) -> boolean
+ predicateExpression: SelectColumnIsNull(col 5:double)
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [6, 2, 4, 1, 16]
- selectExpressions: VectorCoalesce(columns [12, 6, 13, 14, 15])(children: ConstantVectorExpression(val null) -> 12:string, col 6, CastLongToString(col 2) -> 13:String, VectorUDFAdaptor(null(cfloat)) -> 14:string, CastLongToString(col 1) -> 15:String) -> 16:string
+ projectedOutputColumnNums: [6, 2, 4, 1, 17]
+ selectExpressions: VectorCoalesce(columns [13, 6, 14, 15, 16])(children: ConstantVectorExpression(val null) -> 13:string, col 6:string, CastLongToString(col 2:int) -> 14:string, VectorUDFAdaptor(null(cfloat)) -> 15:string, CastLongToString(col 1:smallint) -> 16:string) -> 17:string
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
native: true
@@ -47,7 +46,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: true
@@ -57,7 +57,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -65,14 +64,14 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
Limit Vectorization:
className: VectorLimitOperator
native: true
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [5, 0, 1, 2, 3, 4]
+ projectedOutputColumnNums: [5, 0, 1, 2, 3, 4]
selectExpressions: ConstantVectorExpression(val null) -> 5:double
File Sink Vectorization:
className: VectorFileSinkOperator
@@ -137,16 +136,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNull(col 0) -> boolean
+ predicateExpression: SelectColumnIsNull(col 0:tinyint)
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [5, 2, 15]
- selectExpressions: VectorCoalesce(columns [12, 14, 13])(children: ConstantVectorExpression(val null) -> 12:double, DoubleColAddDoubleColumn(col 5, col 13)(children: FuncLog2LongToDouble(col 2) -> 13:double) -> 14:double, ConstantVectorExpression(val 0.0) -> 13:double) -> 15:double
+ projectedOutputColumnNums: [5, 2, 16]
+ selectExpressions: VectorCoalesce(columns [13, 15, 14])(children: ConstantVectorExpression(val null) -> 13:double, DoubleColAddDoubleColumn(col 5:double, col 14:double)(children: FuncLog2LongToDouble(col 2:int) -> 14:double) -> 15:double, ConstantVectorExpression(val 0.0) -> 14:double) -> 16:double
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
native: true
@@ -156,7 +154,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -166,7 +165,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -174,14 +172,14 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2]
+ projectedOutputColumnNums: [0, 1, 2]
Limit Vectorization:
className: VectorLimitOperator
native: true
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [3, 0, 1, 2]
+ projectedOutputColumnNums: [3, 0, 1, 2]
selectExpressions: ConstantVectorExpression(val null) -> 3:tinyint
File Sink Vectorization:
className: VectorFileSinkOperator
@@ -244,16 +242,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 4) -> boolean, SelectColumnIsNull(col 3) -> boolean) -> boolean
+ predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 4:float), SelectColumnIsNull(col 3:bigint))
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12, 13, 14]
- selectExpressions: ConstantVectorExpression(val null) -> 12:float, ConstantVectorExpression(val null) -> 13:bigint, ConstantVectorExpression(val 0.0) -> 14:double
+ projectedOutputColumnNums: [13, 14, 15]
+ selectExpressions: ConstantVectorExpression(val null) -> 13:float, ConstantVectorExpression(val null) -> 14:bigint, ConstantVectorExpression(val 0.0) -> 15:float
Limit Vectorization:
className: VectorLimitOperator
native: true
@@ -265,7 +262,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -330,16 +328,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: FilterExprOrExpr(children: SelectColumnIsNotNull(col 8) -> boolean, SelectColumnIsNotNull(col 9) -> boolean) -> boolean
+ predicateExpression: FilterExprOrExpr(children: SelectColumnIsNotNull(col 8:timestamp), SelectColumnIsNotNull(col 9:timestamp))
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [8, 9, 12]
- selectExpressions: VectorCoalesce(columns [8, 9])(children: col 8, col 9) -> 12:timestamp
+ projectedOutputColumnNums: [8, 9, 13]
+ selectExpressions: VectorCoalesce(columns [8, 9])(children: col 8:timestamp, col 9:timestamp) -> 13:timestamp
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
native: true
@@ -349,7 +346,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -359,7 +357,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -367,7 +364,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2]
+ projectedOutputColumnNums: [0, 1, 2]
Limit Vectorization:
className: VectorLimitOperator
native: true
@@ -432,16 +429,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 4) -> boolean, SelectColumnIsNull(col 3) -> boolean) -> boolean
+ predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 4:float), SelectColumnIsNull(col 3:bigint))
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12, 13, 14]
- selectExpressions: ConstantVectorExpression(val null) -> 12:float, ConstantVectorExpression(val null) -> 13:bigint, ConstantVectorExpression(val null) -> 14:float
+ projectedOutputColumnNums: [13, 14, 15]
+ selectExpressions: ConstantVectorExpression(val null) -> 13:float, ConstantVectorExpression(val null) -> 14:bigint, ConstantVectorExpression(val null) -> 15:float
Limit Vectorization:
className: VectorLimitOperator
native: true
@@ -453,7 +449,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -514,16 +511,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNull(col 3) -> boolean
+ predicateExpression: SelectColumnIsNull(col 3:bigint)
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12, 0, 14]
- selectExpressions: ConstantVectorExpression(val null) -> 12:bigint, VectorCoalesce(columns [13, 0])(children: ConstantVectorExpression(val null) -> 13:bigint, col 0) -> 14:bigint
+ projectedOutputColumnNums: [13, 0, 15]
+ selectExpressions: ConstantVectorExpression(val null) -> 13:bigint, VectorCoalesce(columns [14, 0])(children: ConstantVectorExpression(val null) -> 14:bigint, col 0:tinyint) -> 15:bigint
Limit Vectorization:
className: VectorLimitOperator
native: true
@@ -535,7 +531,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
index 11825d0..2a95ed0 100644
--- a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
@@ -53,12 +53,6 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col1)
- Group By Vectorization:
- groupByMode: HASH
- vectorOutput: false
- native: false
- vectorProcessingMode: NONE
- projectedOutputColumns: null
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
@@ -76,12 +70,6 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- Group By Vectorization:
- groupByMode: MERGEPARTIAL
- vectorOutput: false
- native: false
- vectorProcessingMode: NONE
- projectedOutputColumns: null
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
@@ -215,27 +203,25 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Select Operator
expressions: str2 (type: string), UDFToInteger(COALESCE(str1,0)) (type: int)
outputColumnNames: _col0, _col1
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 4]
- selectExpressions: CastStringToLong(col 3)(children: VectorCoalesce(columns [0, 2])(children: col 0, ConstantVectorExpression(val 0) -> 2:string) -> 3:string) -> 4:int
+ projectedOutputColumnNums: [1, 5]
+ selectExpressions: CastStringToLong(col 4:string)(children: VectorCoalesce(columns [0, 3])(children: col 0:string, ConstantVectorExpression(val 0) -> 3:string) -> 4:string) -> 5:int
Statistics: Num rows: 4 Data size: 1472 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col1)
Group By Vectorization:
- aggregators: VectorUDAFSumLong(col 4) -> bigint
+ aggregators: VectorUDAFSumLong(col 5:int) -> bigint
className: VectorGroupByOperator
groupByMode: HASH
- vectorOutput: true
- keyExpressions: col 1
+ keyExpressions: col 1:string
native: false
vectorProcessingMode: HASH
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
@@ -255,7 +241,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -265,7 +252,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -273,14 +259,13 @@ STAGE PLANS:
Group By Operator
aggregations: sum(VALUE._col0)
Group By Vectorization:
- aggregators: VectorUDAFSumLong(col 1) -> bigint
+ aggregators: VectorUDAFSumLong(col 1:bigint) -> bigint
className: VectorGroupByOperator
groupByMode: MERGEPARTIAL
- vectorOutput: true
- keyExpressions: col 0
+ keyExpressions: col 0:string
native: false
vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
@@ -291,8 +276,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 2]
- selectExpressions: RoundWithNumDigitsDoubleToDouble(col 3, decimalPlaces 2)(children: DoubleColDivideDoubleScalar(col 2, val 60.0)(children: CastLongToDouble(col 1) -> 2:double) -> 3:double) -> 2:double
+ projectedOutputColumnNums: [0, 2]
+ selectExpressions: RoundWithNumDigitsDoubleToDouble(col 3, decimalPlaces 2)(children: DoubleColDivideDoubleScalar(col 2:double, val 60.0)(children: CastLongToDouble(col 1:bigint) -> 2:double) -> 3:double) -> 2:double
Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -355,15 +340,14 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Select Operator
expressions: COALESCE(str1,0) (type: string)
outputColumnNames: _col0
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [3]
- selectExpressions: VectorCoalesce(columns [0, 2])(children: col 0, ConstantVectorExpression(val 0) -> 2:string) -> 3:string
+ projectedOutputColumnNums: [4]
+ selectExpressions: VectorCoalesce(columns [0, 3])(children: col 0:string, ConstantVectorExpression(val 0) -> 3:string) -> 4:string
Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -380,7 +364,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out
index 2268a15..5065c72 100644
--- a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out
@@ -93,14 +93,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 10872 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: str (type: string), mp (type: map<string,string>), lst (type: array<string>), strct (type: struct<a:string,b:string>), val (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
Statistics: Num rows: 3 Data size: 10872 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -117,7 +117,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -127,6 +128,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3, 4]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -173,14 +175,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: str (type: string)
outputColumnNames: _col0
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -197,7 +199,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -207,6 +210,7 @@ STAGE PLANS:
includeColumns: [0]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -253,14 +257,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 9768 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: strct (type: struct<a:string,b:string>), mp (type: map<string,string>), lst (type: array<string>)
outputColumnNames: _col0, _col1, _col2
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [3, 1, 2]
+ projectedOutputColumnNums: [3, 1, 2]
Statistics: Num rows: 3 Data size: 9768 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -277,7 +281,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -287,6 +292,7 @@ STAGE PLANS:
includeColumns: [1, 2, 3]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -333,14 +339,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: lst (type: array<string>), str (type: string)
outputColumnNames: _col0, _col1
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [2, 0]
+ projectedOutputColumnNums: [2, 0]
Statistics: Num rows: 3 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -357,7 +363,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -367,6 +374,7 @@ STAGE PLANS:
includeColumns: [0, 2]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -413,14 +421,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 3312 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: mp (type: map<string,string>), str (type: string)
outputColumnNames: _col0, _col1
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 0]
+ projectedOutputColumnNums: [1, 0]
Statistics: Num rows: 3 Data size: 3312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -437,7 +445,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -447,6 +456,7 @@ STAGE PLANS:
includeColumns: [0, 1]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -493,14 +503,14 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 1800 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: strct (type: struct<a:string,b:string>), str (type: string)
outputColumnNames: _col0, _col1
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [3, 0]
+ projectedOutputColumnNums: [3, 0]
Statistics: Num rows: 3 Data size: 1800 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -517,7 +527,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -527,6 +538,7 @@ STAGE PLANS:
includeColumns: [0, 3]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Stage: Stage-0
Fetch Operator
@@ -589,7 +601,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Could not vectorize expression (mode = PROJECTION): Column[strct].b
+ notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): Column[strct].b
vectorized: false
Stage: Stage-0
@@ -645,30 +657,31 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: str (type: string), mp (type: map<string,string>), lst (type: array<string>), strct (type: struct<a:string,b:string>)
outputColumnNames: _col0, _col1, _col2, _col3
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ projectedOutputColumnNums: [0, 1, 2, 3]
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
- keyColumns: []
+ keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: [0, 1, 2, 3]
+ valueColumnNums: [0, 1, 2, 3]
Statistics: Num rows: 1 Data size: 3440 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: map<string,string>), _col2 (type: array<string>), _col3 (type: struct<a:string,b:string>)
Execution mode: vectorized, llap
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -678,6 +691,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Map 3
Map Operator Tree:
TableScan
@@ -685,27 +699,28 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: []
+ projectedOutputColumnNums: []
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
- keyColumns: []
+ keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: []
+ valueColumnNums: []
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -715,6 +730,7 @@ STAGE PLANS:
includeColumns: []
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Map 4
Map Operator Tree:
TableScan
@@ -722,27 +738,28 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: []
+ projectedOutputColumnNums: []
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
- keyColumns: []
+ keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: []
+ valueColumnNums: []
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -752,6 +769,7 @@ STAGE PLANS:
includeColumns: []
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Map 5
Map Operator Tree:
TableScan
@@ -759,30 +777,31 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
+ vectorizationSchemaColumns: [0:key:string, 1:value:string, 2:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
- keyColumns: []
+ keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: [0]
+ valueColumnNums: [0]
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
Execution mode: vectorized, llap
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -792,6 +811,7 @@ STAGE PLANS:
includeColumns: [0]
dataColumns: key:string, value:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -885,12 +905,12 @@ STAGE PLANS:
Statistics: Num rows: 13503 Data size: 15460932 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: []
+ projectedOutputColumnNums: []
Statistics: Num rows: 13503 Data size: 15460932 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
@@ -898,10 +918,9 @@ STAGE PLANS:
aggregators: VectorUDAFCountStar(*) -> bigint
className: VectorGroupByOperator
groupByMode: HASH
- vectorOutput: true
native: false
vectorProcessingMode: HASH
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -909,17 +928,18 @@ STAGE PLANS:
sort order:
Reduce Sink Vectorization:
className: VectorReduceSinkEmptyKeyOperator
- keyColumns: []
+ keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: [0]
+ valueColumnNums: [0]
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
Execution mode: vectorized, llap
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -929,6 +949,7 @@ STAGE PLANS:
includeColumns: []
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -936,7 +957,6 @@ STAGE PLANS:
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
reduceColumnNullOrder:
reduceColumnSortOrder:
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -944,17 +964,17 @@ STAGE PLANS:
dataColumnCount: 1
dataColumns: VALUE._col0:bigint
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 0) -> bigint
+ aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint
className: VectorGroupByOperator
groupByMode: MERGEPARTIAL
- vectorOutput: true
native: false
vectorProcessingMode: GLOBAL
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1104,26 +1124,25 @@ STAGE PLANS:
Statistics: Num rows: 13503 Data size: 4721072 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4]
+ vectorizationSchemaColumns: [0:str:string, 1:mp:map<string,string>, 2:lst:array<string>, 3:strct:struct<a:string,b:string>, 4:val:string, 5:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: str (type: string), val (type: string)
outputColumnNames: str, val
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 4]
+ projectedOutputColumnNums: [0, 4]
Statistics: Num rows: 13503 Data size: 4721072 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(val)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 4) -> bigint
+ aggregators: VectorUDAFCount(col 4:string) -> bigint
className: VectorGroupByOperator
groupByMode: HASH
- vectorOutput: true
- keyExpressions: col 0
+ keyExpressions: col 0:string
native: false
vectorProcessingMode: HASH
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
keys: str (type: string)
mode: hash
outputColumnNames: _col0, _col1
@@ -1134,10 +1153,10 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Reduce Sink Vectorization:
className: VectorReduceSinkStringOperator
- keyColumns: [0]
+ keyColumnNums: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: [1]
+ valueColumnNums: [1]
Statistics: Num rows: 13503 Data size: 4721072 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
@@ -1145,7 +1164,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -1155,6 +1175,7 @@ STAGE PLANS:
includeColumns: [0, 4]
dataColumns: str:string, mp:map<string,string>, lst:array<string>, strct:struct<a:string,b:string>, val:string
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -1162,7 +1183,6 @@ STAGE PLANS:
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
reduceColumnNullOrder: a
reduceColumnSortOrder: +
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -1170,18 +1190,18 @@ STAGE PLANS:
dataColumnCount: 2
dataColumns: KEY._col0:string, VALUE._col0:bigint
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 1) -> bigint
+ aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint
className: VectorGroupByOperator
groupByMode: MERGEPARTIAL
- vectorOutput: true
- keyExpressions: col 0
+ keyExpressions: col 0:string
native: false
vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
@@ -1266,7 +1286,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Could not vectorize expression (mode = PROJECTION): Column[strct].b
+ notVectorizedReason: SELECT operator: Could not vectorize expression (mode = PROJECTION): Column[strct].b
vectorized: false
Reducer 2
Execution mode: vectorized, llap
@@ -1275,7 +1295,6 @@ STAGE PLANS:
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
reduceColumnNullOrder: a
reduceColumnSortOrder: +
- groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
@@ -1283,18 +1302,18 @@ STAGE PLANS:
dataColumnCount: 2
dataColumns: KEY._col0:string, VALUE._col0:bigint
partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 1) -> bigint
+ aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint
className: VectorGroupByOperator
groupByMode: MERGEPARTIAL
- vectorOutput: true
- keyExpressions: col 0
+ keyExpressions: col 0:string
native: false
vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumns: [0]
+ projectedOutputColumnNums: [0]
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out
index 4962139..ee9e40a 100644
--- a/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_complex_join.q.out
@@ -47,12 +47,11 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 2) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 2:int)
predicate: cint is not null (type: boolean)
Statistics: Num rows: 9173 Data size: 2309110 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
@@ -61,7 +60,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Statistics: Num rows: 9173 Data size: 2309110 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
@@ -92,7 +91,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -104,12 +104,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0, 1]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 0) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 0:int)
predicate: a is not null (type: boolean)
Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -118,7 +117,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 1]
Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
@@ -135,7 +134,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
usesVectorUDFAdaptor: false
@@ -223,12 +223,11 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
- projectedOutputColumns: [0]
Filter Operator
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsNotNull(col 0) -> boolean
+ predicateExpression: SelectColumnIsNotNull(col 0:int)
predicate: a is not null (type: boolean)
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
@@ -251,7 +250,7 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 1]
+ projectedOutputColumnNums: [0, 2]
Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -268,7 +267,8 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -293,7 +293,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Predicate expression for FILTER operator: org.apache.hadoop.hive.ql.metadata.HiveException: Unexpected hive type name array<int>
+ notVectorizedReason: FILTER operator: Unexpected hive type name array<int>
vectorized: false
Stage: Stage-0