You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/10/04 19:37:38 UTC
[2/3] hive git commit: HIVE-20563: Vectorization: CASE WHEN
expression fails when THEN/ELSE type and result type are different (Matt
McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
index c64adbf..b11ad87 100644
--- a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
@@ -140,23 +140,46 @@ STAGE PLANS:
TableScan
alias: timestamps
Statistics: Num rows: 51 Data size: 12597 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:cdate:date, 1:ctimestamp1:timestamp, 2:stimestamp1:string, 3:ctimestamp2:timestamp, 4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END (type: string), CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2
< TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN (null) ELSE (null) END (type: string), if((ctimestamp1 < TIMESTAMP'1974-10-04 17:21:03.989'), year(ctimestamp1), year(ctimestamp2)) (type: int), CASE WHEN ((stimestamp1 like '%19%')) THEN (stimestamp1) ELSE (TIMESTAMP'2018-03-08 23:04:59') END (type: string), if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)) (type: int), if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null) (type: int), if(((UDFToDouble(ctimestamp1) % 500.0D) > 100.0D), date_add(cdate, 1), date_add(cdate, 365)) (type: date), stimestamp1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 3, 9, 10, 11, 8, 12, 7, 6, 17, 2]
+ selectExpressions: VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'1800-12-31 00:00:00')) THEN ('1800s or Earlier') WHEN ((ctimestamp2 < TIMESTAMP'1900-01-01 00:00:00')) THEN ('1900s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE ('Unknown') END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 5:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 6:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean) -> 9:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIME
STAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN ('Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN ('Early 2010s') ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 5:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean) -> 10:string, VectorUDFAdaptor(CASE WHEN ((ctimestamp2 <= TIMESTAMP'2000-12-31 23:59:59.999999999')) THEN ('Old') WHEN ((ctimestamp2 < TIMESTAMP'2006-01-01 00:00:00')) THEN ('Early 2000s') WHEN (ctimestamp2 BETWEEN TIMESTAMP'2006-01-01 00:00:00' AND TIMESTAMP'2010-12-31 23:59:59.999999999') THEN (
'Late 2000s') WHEN ((ctimestamp2 <= TIMESTAMP'2015-12-31 23:59:59.999999999')) THEN (null) ELSE (null) END)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 5:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean) -> 11:string, IfExprLongColumnLongColumn(col 5:boolean, col 6:int, col 7:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 5:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 6:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 7:int) -> 8:int, VectorUDFAdaptor(CASE WHEN ((stimestamp1 like '%19%')) THEN (stimestamp1) ELSE (TIMESTAMP'2018-03-08 23:04:59') END)(children: SelectStringColLikeString
Scalar(col 2:string) -> 5:boolean) -> 12:string, VectorUDFAdaptor(if((ctimestamp1 = TIMESTAMP'2021-09-24 03:18:32.413655165'), null, minute(ctimestamp1)))(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 5:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 6:int) -> 7:int, VectorUDFAdaptor(if(((ctimestamp2 >= TIMESTAMP'5344-10-04 18:40:08.165') and (ctimestamp2 < TIMESTAMP'6631-11-13 16:31:29.702202248')), minute(ctimestamp1), null))(children: ColAndCol(col 5:boolean, col 6:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 5:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 6:boolean) -> 13:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 5:int) -> 6:int, IfExprLongColumnLongColumn(col 5:boolean, col 13:date, col 16:date)(children: DoubleColGreaterDoubleScalar(col 15:double, val 100.0)(children: D
oubleColModuloDoubleScalar(col 14:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 14:double) -> 15:double) -> 5:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 13:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 16:date) -> 17:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp)
sort order: +++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 1:timestamp, 2:string, 3:timestamp
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:string, 10:string, 11:string, 8:int, 12:string, 7:int, 6:int, 17:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date)
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected hive type name void
- vectorized: false
+ allNative: true
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: cdate:date, ctimestamp1:timestamp, stimestamp1:string, ctimestamp2:timestamp
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, bigint, double, double, bigint, bigint]
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -415,8 +438,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [1, 3, 10, 12, 13, 14, 11, 7, 16, 23, 2]
- selectExpressions: IfExprStringScalarStringGroupColumn(col 5:boolean, val 1800s or Earliercol 9:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val 1900scol 10:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 9:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprStringScalarStringScalar(col 8:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean) -> 9:string) -> 10:string) -> 9:string) -> 10:string, IfExprStringScalarStringGroupColumn(col 5:boolean, val Oldcol 11:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2
000-12-31 23:59:59.999999999) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val Early 2000scol 12:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 11:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprColumnNull(col 8:boolean, col 9:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean, ConstantVectorExpression(val Early 2010s) -> 9:string) -> 11:string) -> 12:string) -> 11:string) -> 12:string, IfExprStringScalarStringGroupColumn(col 5:boolean, val Oldcol 11:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val Early 2000scol 13:string)(children: TimestampCol
LessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 11:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprNullNull(null, null) -> 11:string) -> 13:string) -> 11:string) -> 13:string, IfExprLongColumnLongColumn(col 5:boolean, col 6:int, col 7:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 5:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 6:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 7:int) -> 14:int, VectorUDFAdaptor(CASE WHEN ((stimestamp1 like '%19%')) THEN (stimestamp1) ELSE (TIMESTAMP'2018-03-08 23:04:59') END)(children: SelectStringColLikeStringScalar(col 2:string) -> 5:boolean) -> 11:string, IfExprNullColumn(col 5:boolean, null, col 6)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:3
2.413655165) -> 5:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 6:int) -> 7:int, IfExprColumnNull(col 17:boolean, col 15:int, null)(children: ColAndCol(col 15:boolean, col 16:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 15:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 16:boolean) -> 17:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 15:int) -> 16:int, IfExprLongColumnLongColumn(col 20:boolean, col 21:date, col 22:date)(children: DoubleColGreaterDoubleScalar(col 19:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 18:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 18:double) -> 19:double) -> 20:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 21:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 22:date) -> 23:date
+ projectedOutputColumnNums: [1, 3, 10, 12, 13, 14, 15, 7, 17, 24, 2]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 5:boolean, val 1800s or Earliercol 9:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val 1900scol 10:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 9:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprStringScalarStringScalar(col 8:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean) -> 9:string) -> 10:string) -> 9:string) -> 10:string, IfExprStringScalarStringGroupColumn(col 5:boolean, val Oldcol 11:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2
000-12-31 23:59:59.999999999) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val Early 2000scol 12:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 11:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprColumnNull(col 8:boolean, col 9:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 8:boolean, ConstantVectorExpression(val Early 2010s) -> 9:string) -> 11:string) -> 12:string) -> 11:string) -> 12:string, IfExprStringScalarStringGroupColumn(col 5:boolean, val Oldcol 11:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 5:boolean, IfExprStringScalarStringGroupColumn(col 6:boolean, val Early 2000scol 13:string)(children: TimestampCol
LessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 6:boolean, IfExprStringScalarStringGroupColumn(col 7:boolean, val Late 2000scol 11:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 7:boolean, IfExprNullNull(null, null) -> 11:string) -> 13:string) -> 11:string) -> 13:string, IfExprLongColumnLongColumn(col 5:boolean, col 6:int, col 7:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 5:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 6:int, VectorUDFYearTimestamp(col 3:timestamp, field YEAR) -> 7:int) -> 14:int, IfExprStringGroupColumnStringGroupColumn(col 5:boolean, col 2:string, col 2:string)(children: SelectStringColLikeStringScalar(col 2:string) -> 5:boolean, VectorUDFAdaptor(CAST( TIMESTAMP'2018-03-08 23:04:59' AS STRING)) -> 11:string) -> 15:string, IfExprNullColumn(col 5:boolean, null, col 6)(children: TimestampColEqualTimestam
pScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 5:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 6:int) -> 7:int, IfExprColumnNull(col 18:boolean, col 16:int, null)(children: ColAndCol(col 16:boolean, col 17:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 16:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 17:boolean) -> 18:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 16:int) -> 17:int, IfExprLongColumnLongColumn(col 21:boolean, col 22:date, col 23:date)(children: DoubleColGreaterDoubleScalar(col 20:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 19:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 19:double) -> 20:double) -> 21:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 22:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 23:date) -> 24:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp)
@@ -426,7 +449,7 @@ STAGE PLANS:
keyColumns: 1:timestamp, 2:string, 3:timestamp
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: 10:string, 12:string, 13:string, 14:int, 11:string, 7:int, 16:int, 23:date
+ valueColumns: 10:string, 12:string, 13:string, 14:int, 15:string, 7:int, 17:int, 24:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date)
Execution mode: vectorized, llap
@@ -445,7 +468,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3]
dataColumns: cdate:date, ctimestamp1:timestamp, stimestamp1:string, ctimestamp2:timestamp
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, double, double, bigint, bigint, bigint, bigint]
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, string, string, bigint, string, bigint, bigint, bigint, double, double, bigint, bigint, bigint, bigint]
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -704,8 +727,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [1, 3, 15, 26, 36, 40, 42, 44, 46, 53, 2]
- selectExpressions: IfExprColumnCondExpr(col 5:boolean, col 6:stringcol 14:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 5:boolean, ConstantVectorExpression(val 1800s or Earlier) -> 6:string, IfExprColumnCondExpr(col 7:boolean, col 8:stringcol 13:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, ConstantVectorExpression(val 1900s) -> 8:string, IfExprColumnCondExpr(col 9:boolean, col 10:stringcol 12:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 9:boolean, ConstantVectorExpression(val Late 2000s) -> 10:string, IfExprStringScalarStringScalar(col 11:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 11:boolean) -> 12:string) -> 13:string) -> 14:string) -> 15:string, IfExprColumnCondExpr(co
l 11:boolean, col 16:stringcol 25:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 11:boolean, ConstantVectorExpression(val Old) -> 16:string, IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 24:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 17:boolean, ConstantVectorExpression(val Early 2000s) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 23:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 19:boolean, ConstantVectorExpression(val Late 2000s) -> 20:string, IfExprColumnNull(col 21:boolean, col 22:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 21:boolean, ConstantVectorExpression(val Early 2010s) -> 22:string) -> 23:string) -> 24:string) -> 25:string) -> 26:string, IfExprColumnCondExpr(col 27:boolean, col 28
:stringcol 35:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 27:boolean, ConstantVectorExpression(val Old) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 34:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 29:boolean, ConstantVectorExpression(val Early 2000s) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 33:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 31:boolean, ConstantVectorExpression(val Late 2000s) -> 32:string, IfExprNullNull(null, null) -> 33:string) -> 34:string) -> 35:string) -> 36:string, IfExprCondExprCondExpr(col 37:boolean, col 38:intcol 39:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 37:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 38:int, VectorUDFYearTimestamp(col 3:time
stamp, field YEAR) -> 39:int) -> 40:int, VectorUDFAdaptor(CASE WHEN ((stimestamp1 like '%19%')) THEN (stimestamp1) ELSE (TIMESTAMP'2018-03-08 23:04:59') END)(children: SelectStringColLikeStringScalar(col 2:string) -> 41:boolean) -> 42:string, IfExprNullCondExpr(col 41:boolean, null, col 43:int)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 41:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 43:int) -> 44:int, IfExprCondExprNull(col 47:boolean, col 45:int, null)(children: ColAndCol(col 45:boolean, col 46:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 45:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 46:boolean) -> 47:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 45:int) -> 46:int, IfExprCondExprCondExpr(col 50:boolean, col 51:datecol 52:date)(children: DoubleColGreaterDoubleScalar(col
49:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 48:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 48:double) -> 49:double) -> 50:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 51:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 52:date) -> 53:date
+ projectedOutputColumnNums: [1, 3, 15, 26, 36, 40, 43, 46, 48, 55, 2]
+ selectExpressions: IfExprColumnCondExpr(col 5:boolean, col 6:stringcol 14:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 1800-12-31 00:00:00) -> 5:boolean, ConstantVectorExpression(val 1800s or Earlier) -> 6:string, IfExprColumnCondExpr(col 7:boolean, col 8:stringcol 13:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 1900-01-01 00:00:00) -> 7:boolean, ConstantVectorExpression(val 1900s) -> 8:string, IfExprColumnCondExpr(col 9:boolean, col 10:stringcol 12:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 9:boolean, ConstantVectorExpression(val Late 2000s) -> 10:string, IfExprStringScalarStringScalar(col 11:boolean, val Early 2010s, val Unknown)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 11:boolean) -> 12:string) -> 13:string) -> 14:string) -> 15:string, IfExprColumnCondExpr(co
l 11:boolean, col 16:stringcol 25:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 11:boolean, ConstantVectorExpression(val Old) -> 16:string, IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 24:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 17:boolean, ConstantVectorExpression(val Early 2000s) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 23:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 19:boolean, ConstantVectorExpression(val Late 2000s) -> 20:string, IfExprColumnNull(col 21:boolean, col 22:string, null)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2015-12-31 23:59:59.999999999) -> 21:boolean, ConstantVectorExpression(val Early 2010s) -> 22:string) -> 23:string) -> 24:string) -> 25:string) -> 26:string, IfExprColumnCondExpr(col 27:boolean, col 28
:stringcol 35:string)(children: TimestampColLessEqualTimestampScalar(col 3:timestamp, val 2000-12-31 23:59:59.999999999) -> 27:boolean, ConstantVectorExpression(val Old) -> 28:string, IfExprColumnCondExpr(col 29:boolean, col 30:stringcol 34:string)(children: TimestampColLessTimestampScalar(col 3:timestamp, val 2006-01-01 00:00:00) -> 29:boolean, ConstantVectorExpression(val Early 2000s) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 33:string)(children: TimestampColumnBetween(col 3:timestamp, left 2005-12-31 16:00:00.0, right 2010-12-31 15:59:59.999999999) -> 31:boolean, ConstantVectorExpression(val Late 2000s) -> 32:string, IfExprNullNull(null, null) -> 33:string) -> 34:string) -> 35:string) -> 36:string, IfExprCondExprCondExpr(col 37:boolean, col 38:intcol 39:int)(children: TimestampColLessTimestampScalar(col 1:timestamp, val 1974-10-04 17:21:03.989) -> 37:boolean, VectorUDFYearTimestamp(col 1:timestamp, field YEAR) -> 38:int, VectorUDFYearTimestamp(col 3:time
stamp, field YEAR) -> 39:int) -> 40:int, IfExprColumnCondExpr(col 41:boolean, col 2:stringcol 42:string)(children: SelectStringColLikeStringScalar(col 2:string) -> 41:boolean, col 2:string, VectorUDFAdaptor(CAST( TIMESTAMP'2018-03-08 23:04:59' AS STRING)) -> 42:string) -> 43:string, IfExprNullCondExpr(col 44:boolean, null, col 45:int)(children: TimestampColEqualTimestampScalar(col 1:timestamp, val 2021-09-24 03:18:32.413655165) -> 44:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 45:int) -> 46:int, IfExprCondExprNull(col 49:boolean, col 47:int, null)(children: ColAndCol(col 47:boolean, col 48:boolean)(children: TimestampColGreaterEqualTimestampScalar(col 3:timestamp, val 5344-10-04 18:40:08.165) -> 47:boolean, TimestampColLessTimestampScalar(col 3:timestamp, val 6631-11-13 16:31:29.702202248) -> 48:boolean) -> 49:boolean, VectorUDFMinuteTimestamp(col 1:timestamp, field MINUTE) -> 47:int) -> 48:int, IfExprCondExprCondExpr(col 52:boolean, col 53:datecol 54:date)(c
hildren: DoubleColGreaterDoubleScalar(col 51:double, val 100.0)(children: DoubleColModuloDoubleScalar(col 50:double, val 500.0)(children: CastTimestampToDouble(col 1:timestamp) -> 50:double) -> 51:double) -> 52:boolean, VectorUDFDateAddColScalar(col 0:date, val 1) -> 53:date, VectorUDFDateAddColScalar(col 0:date, val 365) -> 54:date) -> 55:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col10 (type: string), _col1 (type: timestamp)
@@ -715,7 +738,7 @@ STAGE PLANS:
keyColumns: 1:timestamp, 2:string, 3:timestamp
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumns: 15:string, 26:string, 36:string, 40:int, 42:string, 44:int, 46:int, 53:date
+ valueColumns: 15:string, 26:string, 36:string, 40:int, 43:string, 46:int, 48:int, 55:date
Statistics: Num rows: 51 Data size: 50745 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: date)
Execution mode: vectorized, llap
@@ -734,7 +757,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3]
dataColumns: cdate:date, ctimestamp1:timestamp, stimestamp1:string, ctimestamp2:timestamp
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, bigint, bigint]
+ scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, string, string, bigint, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, bigint, bigint]
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out
new file mode 100644
index 0000000..8e07710
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out
@@ -0,0 +1,616 @@
+PREHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNull(col 5:double)
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [6, 2, 4, 1, 24]
+ selectExpressions: IfExprColumnCondExpr(col 13:boolean, col 6:stringcol 23:string)(children: IsNotNull(col 6:string) -> 13:boolean, col 6:string, IfExprCondExprCondExpr(col 14:boolean, col 15:stringcol 22:string)(children: IsNotNull(col 2:int) -> 14:boolean, CastLongToString(col 2:int) -> 15:string, IfExprCondExprCondExpr(col 16:boolean, col 17:stringcol 21:string)(children: IsNotNull(col 4:float) -> 16:boolean, CastFloatToString(col 4:float) -> 17:string, IfExprCondExprColumn(col 18:boolean, col 19:string, col 20:string)(children: IsNotNull(col 1:smallint) -> 18:boolean, CastLongToString(col 1:smallint) -> 19:string, ConstantVectorExpression(val none) -> 20:string) -> 21:string) -> 22:string) -> 23:string) -> 24:string
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [5, 0, 1, 2, 3, 4]
+ selectExpressions: ConstantVectorExpression(val null) -> 5:double
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
+PREHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+NULL 00MmJs1fiJp37y60mj4Ej8 -698191930 -51.0 NULL 00MmJs1fiJp37y60mj4Ej8
+NULL 00PafC7v 349566607 -51.0 NULL 00PafC7v
+NULL 00iT08 284688862 -51.0 NULL 00iT08
+NULL 00k3yt70n476d6UQA -391432229 8.0 NULL 00k3yt70n476d6UQA
+NULL 014ILGhXxNY7g02hl0Xw 633097881 11.0 NULL 014ILGhXxNY7g02hl0Xw
+NULL 02VRbSC5I 551634127 8.0 NULL 02VRbSC5I
+NULL 02k5poW73QsWM 891702124 11.0 NULL 02k5poW73QsWM
+NULL 02v8WnLuYDos3Cq -648704945 8.0 NULL 02v8WnLuYDos3Cq
+NULL 02vDyIVT752 388584379 11.0 NULL 02vDyIVT752
+NULL 0333uXvwB3ADRa4aP1h 336245146 8.0 NULL 0333uXvwB3ADRa4aP1h
+NULL 033ffm5082ng0V -941753533 11.0 NULL 033ffm5082ng0V
+NULL 035i4wu42Rs3Uu1ft5K0AOe -947302120 8.0 NULL 035i4wu42Rs3Uu1ft5K0AOe
+NULL 03SnoFNyeHxQ2X -693113839 8.0 NULL 03SnoFNyeHxQ2X
+NULL 03n0QGH 1018006843 11.0 NULL 03n0QGH
+NULL 04Y1mA17 -114647521 -51.0 NULL 04Y1mA17
+NULL 04Yu8RntCU7amJtj -640911032 -51.0 NULL 04Yu8RntCU7amJtj
+NULL 04fq7M416mV7CwI1q 168027481 -51.0 NULL 04fq7M416mV7CwI1q
+NULL 04q7g1Qm8cvCmny4S7r 118167064 -51.0 NULL 04q7g1Qm8cvCmny4S7r
+NULL 04vwGN4a82bd6y 295643033 NULL NULL 04vwGN4a82bd6y
+NULL 04w7DF25lHW4 -981967139 8.0 NULL 04w7DF25lHW4
+PREHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: VectorUDFAdaptor(cdouble is null) -> 13:boolean)
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [6, 2, 4, 1, 18]
+ selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (cint) WHEN (cfloat is not null) THEN (cfloat) WHEN (csmallint is not null) THEN (csmallint) ELSE ('none') END)(children: VectorUDFAdaptor(cstring1 is not null) -> 14:boolean, VectorUDFAdaptor(cint is not null) -> 15:boolean, VectorUDFAdaptor(cfloat is not null) -> 16:boolean, VectorUDFAdaptor(csmallint is not null) -> 17:boolean) -> 18:string
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [5, 0, 1, 2, 3, 4]
+ selectExpressions: ConstantVectorExpression(val null) -> 5:double
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
+PREHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+NULL 00MmJs1fiJp37y60mj4Ej8 -698191930 -51.0 NULL 00MmJs1fiJp37y60mj4Ej8
+NULL 00PafC7v 349566607 -51.0 NULL 00PafC7v
+NULL 00iT08 284688862 -51.0 NULL 00iT08
+NULL 00k3yt70n476d6UQA -391432229 8.0 NULL 00k3yt70n476d6UQA
+NULL 014ILGhXxNY7g02hl0Xw 633097881 11.0 NULL 014ILGhXxNY7g02hl0Xw
+NULL 02VRbSC5I 551634127 8.0 NULL 02VRbSC5I
+NULL 02k5poW73QsWM 891702124 11.0 NULL 02k5poW73QsWM
+NULL 02v8WnLuYDos3Cq -648704945 8.0 NULL 02v8WnLuYDos3Cq
+NULL 02vDyIVT752 388584379 11.0 NULL 02vDyIVT752
+NULL 0333uXvwB3ADRa4aP1h 336245146 8.0 NULL 0333uXvwB3ADRa4aP1h
+NULL 033ffm5082ng0V -941753533 11.0 NULL 033ffm5082ng0V
+NULL 035i4wu42Rs3Uu1ft5K0AOe -947302120 8.0 NULL 035i4wu42Rs3Uu1ft5K0AOe
+NULL 03SnoFNyeHxQ2X -693113839 8.0 NULL 03SnoFNyeHxQ2X
+NULL 03n0QGH 1018006843 11.0 NULL 03n0QGH
+NULL 04Y1mA17 -114647521 -51.0 NULL 04Y1mA17
+NULL 04Yu8RntCU7amJtj -640911032 -51.0 NULL 04Yu8RntCU7amJtj
+NULL 04fq7M416mV7CwI1q 168027481 -51.0 NULL 04fq7M416mV7CwI1q
+NULL 04q7g1Qm8cvCmny4S7r 118167064 -51.0 NULL 04q7g1Qm8cvCmny4S7r
+NULL 04vwGN4a82bd6y 295643033 NULL NULL 04vwGN4a82bd6y
+NULL 04w7DF25lHW4 -981967139 8.0 NULL 04w7DF25lHW4
+PREHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNull(col 5:double)
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [6, 2, 4, 1, 23]
+ selectExpressions: IfExprColumnCondExpr(col 13:boolean, col 6:stringcol 22:string)(children: IsNotNull(col 6:string) -> 13:boolean, col 6:string, IfExprCondExprCondExpr(col 14:boolean, col 15:stringcol 21:string)(children: IsNotNull(col 2:int) -> 14:boolean, CastLongToString(col 2:int) -> 15:string, IfExprCondExprCondExpr(col 16:boolean, col 17:stringcol 20:string)(children: IsNotNull(col 4:float) -> 16:boolean, CastFloatToString(col 4:float) -> 17:string, IfExprCondExprNull(col 18:boolean, col 19:string, null)(children: IsNotNull(col 1:smallint) -> 18:boolean, CastLongToString(col 1:smallint) -> 19:string) -> 20:string) -> 21:string) -> 22:string) -> 23:string
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [5, 0, 1, 2, 3, 4]
+ selectExpressions: ConstantVectorExpression(val null) -> 5:double
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
+PREHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+NULL 00MmJs1fiJp37y60mj4Ej8 -698191930 -51.0 NULL 00MmJs1fiJp37y60mj4Ej8
+NULL 00PafC7v 349566607 -51.0 NULL 00PafC7v
+NULL 00iT08 284688862 -51.0 NULL 00iT08
+NULL 00k3yt70n476d6UQA -391432229 8.0 NULL 00k3yt70n476d6UQA
+NULL 014ILGhXxNY7g02hl0Xw 633097881 11.0 NULL 014ILGhXxNY7g02hl0Xw
+NULL 02VRbSC5I 551634127 8.0 NULL 02VRbSC5I
+NULL 02k5poW73QsWM 891702124 11.0 NULL 02k5poW73QsWM
+NULL 02v8WnLuYDos3Cq -648704945 8.0 NULL 02v8WnLuYDos3Cq
+NULL 02vDyIVT752 388584379 11.0 NULL 02vDyIVT752
+NULL 0333uXvwB3ADRa4aP1h 336245146 8.0 NULL 0333uXvwB3ADRa4aP1h
+NULL 033ffm5082ng0V -941753533 11.0 NULL 033ffm5082ng0V
+NULL 035i4wu42Rs3Uu1ft5K0AOe -947302120 8.0 NULL 035i4wu42Rs3Uu1ft5K0AOe
+NULL 03SnoFNyeHxQ2X -693113839 8.0 NULL 03SnoFNyeHxQ2X
+NULL 03n0QGH 1018006843 11.0 NULL 03n0QGH
+NULL 04Y1mA17 -114647521 -51.0 NULL 04Y1mA17
+NULL 04Yu8RntCU7amJtj -640911032 -51.0 NULL 04Yu8RntCU7amJtj
+NULL 04fq7M416mV7CwI1q 168027481 -51.0 NULL 04fq7M416mV7CwI1q
+NULL 04q7g1Qm8cvCmny4S7r 118167064 -51.0 NULL 04q7g1Qm8cvCmny4S7r
+NULL 04vwGN4a82bd6y 295643033 NULL NULL 04vwGN4a82bd6y
+NULL 04w7DF25lHW4 -981967139 8.0 NULL 04w7DF25lHW4
+PREHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsTrue(col 13:boolean)(children: VectorUDFAdaptor(cdouble is null) -> 13:boolean)
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [6, 2, 4, 1, 18]
+ selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (cint) WHEN (cfloat is not null) THEN (cfloat) WHEN (csmallint is not null) THEN (csmallint) ELSE (null) END)(children: VectorUDFAdaptor(cstring1 is not null) -> 14:boolean, VectorUDFAdaptor(cint is not null) -> 15:boolean, VectorUDFAdaptor(cfloat is not null) -> 16:boolean, VectorUDFAdaptor(csmallint is not null) -> 17:boolean) -> 18:string
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4]
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [5, 0, 1, 2, 3, 4]
+ selectExpressions: ConstantVectorExpression(val null) -> 5:double
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
+PREHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+NULL 00MmJs1fiJp37y60mj4Ej8 -698191930 -51.0 NULL 00MmJs1fiJp37y60mj4Ej8
+NULL 00PafC7v 349566607 -51.0 NULL 00PafC7v
+NULL 00iT08 284688862 -51.0 NULL 00iT08
+NULL 00k3yt70n476d6UQA -391432229 8.0 NULL 00k3yt70n476d6UQA
+NULL 014ILGhXxNY7g02hl0Xw 633097881 11.0 NULL 014ILGhXxNY7g02hl0Xw
+NULL 02VRbSC5I 551634127 8.0 NULL 02VRbSC5I
+NULL 02k5poW73QsWM 891702124 11.0 NULL 02k5poW73QsWM
+NULL 02v8WnLuYDos3Cq -648704945 8.0 NULL 02v8WnLuYDos3Cq
+NULL 02vDyIVT752 388584379 11.0 NULL 02vDyIVT752
+NULL 0333uXvwB3ADRa4aP1h 336245146 8.0 NULL 0333uXvwB3ADRa4aP1h
+NULL 033ffm5082ng0V -941753533 11.0 NULL 033ffm5082ng0V
+NULL 035i4wu42Rs3Uu1ft5K0AOe -947302120 8.0 NULL 035i4wu42Rs3Uu1ft5K0AOe
+NULL 03SnoFNyeHxQ2X -693113839 8.0 NULL 03SnoFNyeHxQ2X
+NULL 03n0QGH 1018006843 11.0 NULL 03n0QGH
+NULL 04Y1mA17 -114647521 -51.0 NULL 04Y1mA17
+NULL 04Yu8RntCU7amJtj -640911032 -51.0 NULL 04Yu8RntCU7amJtj
+NULL 04fq7M416mV7CwI1q 168027481 -51.0 NULL 04fq7M416mV7CwI1q
+NULL 04q7g1Qm8cvCmny4S7r 118167064 -51.0 NULL 04q7g1Qm8cvCmny4S7r
+NULL 04vwGN4a82bd6y 295643033 NULL NULL 04vwGN4a82bd6y
+NULL 04w7DF25lHW4 -981967139 8.0 NULL 04w7DF25lHW4
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out
index 499337e..36f8f93 100644
--- a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out
@@ -368,4 +368,4 @@ ORDER BY c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) q
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_test_small_n0
#### A masked pattern was here ####
-1252336297085
+787712777374
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
index e4c1d71..deab4d2 100644
--- a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
@@ -134,26 +134,46 @@ STAGE PLANS:
TableScan
alias: student_10_lines
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:age:int, 2:gpa:double, 3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: name (type: string), age (type: int), gpa (type: double), if((age < 40), age, null) (type: int), if((age > 40), TIMESTAMP'2011-01-01 01:01:01', null) (type: timestamp), if((length(name) > 8), name, null) (type: string), if((length(name) < 8), CAST( name AS BINARY), null) (type: binary), if((age > 40), length(name), null) (type: int), if((length(name) > 10), (2.0D * gpa), null) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 5, 6, 8, 9, 10, 12]
+ selectExpressions: VectorUDFAdaptor(if((age < 40), age, null))(children: LongColLessLongScalar(col 1:int, val 40) -> 4:boolean) -> 5:int, VectorUDFAdaptor(if((age > 40), TIMESTAMP'2011-01-01 01:01:01', null))(children: LongColGreaterLongScalar(col 1:int, val 40) -> 4:boolean) -> 6:timestamp, VectorUDFAdaptor(if((length(name) > 8), name, null))(children: LongColGreaterLongScalar(col 4:int, val 8)(children: StringLength(col 0:string) -> 4:int) -> 7:boolean) -> 8:string, VectorUDFAdaptor(if((length(name) < 8), CAST( name AS BINARY), null))(children: LongColLessLongScalar(col 4:int, val 8)(children: StringLength(col 0:string) -> 4:int) -> 7:boolean, col 0:string) -> 9:binary, VectorUDFAdaptor(if((age > 40), length(name), null))(children: LongColGreaterLongScalar(col 1:int, val 40) -> 4:boolean, StringLength(col 0:string) -> 7:int) -> 10:int, VectorUDFAdaptor(if((length(name) > 10), (2.0D * gpa), null))(children: LongColGreaterLongScalar(col 4:int, val 10)(childre
n: StringLength(col 0:string) -> 4:int) -> 7:boolean, DoubleScalarMultiplyDoubleColumn(val 2.0, col 2:double) -> 11:double) -> 12:double
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.insert_a_adaptor
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected hive type name void
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, age:int, gpa:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, timestamp, bigint, string, string, bigint, double, double]
Stage: Stage-2
Dependency Collection
@@ -654,26 +674,46 @@ STAGE PLANS:
TableScan
alias: student_10_lines
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:name:string, 1:age:int, 2:gpa:double, 3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: name (type: string), age (type: int), gpa (type: double), if((age < 40), null, age) (type: int), if((age > 40), null, TIMESTAMP'2011-01-01 01:01:01') (type: timestamp), if((length(name) > 8), null, name) (type: string), if((length(name) < 8), null, CAST( name AS BINARY)) (type: binary), if((age > 40), null, length(name)) (type: int), if((length(name) > 10), null, (2.0D * gpa)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 5, 6, 8, 9, 10, 12]
+ selectExpressions: VectorUDFAdaptor(if((age < 40), null, age))(children: LongColLessLongScalar(col 1:int, val 40) -> 4:boolean) -> 5:int, VectorUDFAdaptor(if((age > 40), null, TIMESTAMP'2011-01-01 01:01:01'))(children: LongColGreaterLongScalar(col 1:int, val 40) -> 4:boolean) -> 6:timestamp, VectorUDFAdaptor(if((length(name) > 8), null, name))(children: LongColGreaterLongScalar(col 4:int, val 8)(children: StringLength(col 0:string) -> 4:int) -> 7:boolean) -> 8:string, VectorUDFAdaptor(if((length(name) < 8), null, CAST( name AS BINARY)))(children: LongColLessLongScalar(col 4:int, val 8)(children: StringLength(col 0:string) -> 4:int) -> 7:boolean, col 0:string) -> 9:binary, VectorUDFAdaptor(if((age > 40), null, length(name)))(children: LongColGreaterLongScalar(col 1:int, val 40) -> 4:boolean, StringLength(col 0:string) -> 7:int) -> 10:int, VectorUDFAdaptor(if((length(name) > 10), null, (2.0D * gpa)))(children: LongColGreaterLongScalar(col 4:int, val 10)(childre
n: StringLength(col 0:string) -> 4:int) -> 7:boolean, DoubleScalarMultiplyDoubleColumn(val 2.0, col 2:double) -> 11:double) -> 12:double
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 12 Data size: 2352 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.insert_b_adaptor
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected hive type name void
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: name:string, age:int, gpa:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, timestamp, bigint, string, string, bigint, double, double]
Stage: Stage-2
Dependency Collection
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
index add166b..d7e7561 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
@@ -698,8 +698,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [8]
- selectExpressions: IfExprDecimal64ScalarDecimal64Column(col 6:boolean, decimal64Val 1, decimalVal 1, col 7:decimal(1,0)/DECIMAL_64)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64
+ projectedOutputColumnNums: [9]
+ selectExpressions: IfExprDecimalColumnColumn(col 6:boolean, col 7:decimal(11,0)col 10:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, ConstantVectorExpression(val 1) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 8:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 8:decimal(11,0)/DECIMAL_64) -> 10:decimal(11,0)) -> 9:decimal(11,0)
Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
@@ -727,7 +727,7 @@ STAGE PLANS:
includeColumns: [0, 1]
dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64]
+ scratchColumnTypeNames: [bigint, decimal(11,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0), decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)]
Stage: Stage-0
Fetch Operator
@@ -783,8 +783,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [8]
- selectExpressions: IfExprDecimal64ColumnDecimal64Scalar(col 6:boolean, col 7:decimal(11,0)/DECIMAL_64, decimal64Val 2, decimalVal 2)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64
+ projectedOutputColumnNums: [9]
+ selectExpressions: IfExprDecimalColumnColumn(col 6:boolean, col 10:decimal(11,0)col 8:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 10:decimal(11,0), ConstantVectorExpression(val 2) -> 8:decimal(11,0)) -> 9:decimal(11,0)
Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
@@ -812,7 +812,7 @@ STAGE PLANS:
includeColumns: [0, 1]
dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64]
+ scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0), decimal(11,0)]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
index a96aca5..fa78d79 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
@@ -691,8 +691,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [8]
- selectExpressions: IfExprDecimal64ScalarDecimal64Column(col 6:boolean, decimal64Val 1, decimalVal 1, col 7:decimal(1,0)/DECIMAL_64)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64
+ projectedOutputColumnNums: [9]
+ selectExpressions: IfExprDecimalColumnColumn(col 6:boolean, col 7:decimal(11,0)col 10:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, ConstantVectorExpression(val 1) -> 7:decimal(11,0), ConvertDecimal64ToDecimal(col 8:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 2, decimalVal 2) -> 8:decimal(11,0)/DECIMAL_64) -> 10:decimal(11,0)) -> 9:decimal(11,0)
Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -719,7 +719,7 @@ STAGE PLANS:
includeColumns: [0, 1]
dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, decimal(1,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64]
+ scratchColumnTypeNames: [bigint, decimal(11,0), decimal(11,0)/DECIMAL_64, bigint, decimal(11,0), decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0)]
Stage: Stage-0
Fetch Operator
@@ -775,8 +775,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [8]
- selectExpressions: IfExprDecimal64ColumnDecimal64Scalar(col 6:boolean, col 7:decimal(11,0)/DECIMAL_64, decimal64Val 2, decimalVal 2)(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 8:decimal(11,0)/DECIMAL_64
+ projectedOutputColumnNums: [9]
+ selectExpressions: IfExprDecimalColumnColumn(col 6:boolean, col 10:decimal(11,0)col 8:decimal(11,0))(children: Decimal64ColEqualDecimal64Scalar(col 0:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 6:boolean, ConvertDecimal64ToDecimal(col 7:decimal(11,0)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,0)/DECIMAL_64, decimal64Val 1, decimalVal 1) -> 7:decimal(11,0)/DECIMAL_64) -> 10:decimal(11,0), ConstantVectorExpression(val 2) -> 8:decimal(11,0)) -> 9:decimal(11,0)
Statistics: Num rows: 3 Data size: 672 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -803,7 +803,7 @@ STAGE PLANS:
includeColumns: [0, 1]
dataColumns: member:decimal(10,0)/DECIMAL_64, attr:decimal(10,0)/DECIMAL_64
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(1,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0)/DECIMAL_64]
+ scratchColumnTypeNames: [bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), bigint, decimal(11,0)/DECIMAL_64, decimal(11,0), decimal(11,0), decimal(11,0)]
Stage: Stage-0
Fetch Operator