You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/06/22 23:40:47 UTC
[11/34] hive git commit: HIVE-16589: Vectorization: Support Complex
Types and GroupBy modes PARTIAL2, FINAL, and COMPLETE for AVG,
VARIANCE (Matt McCline, reviewed by Jason Dere)
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_10.q.out b/ql/src/test/results/clientpositive/spark/vectorization_10.q.out
index 9dad4c4..4e9cce3 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_10.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_10.q.out
@@ -1,3 +1,120 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cdouble,
+ ctimestamp1,
+ ctinyint,
+ cboolean1,
+ cstring1,
+ (-(cdouble)),
+ (cdouble + csmallint),
+ ((cdouble + csmallint) % 33),
+ (-(cdouble)),
+ (ctinyint % cdouble),
+ (ctinyint % csmallint),
+ (-(cdouble)),
+ (cbigint * (ctinyint % csmallint)),
+ (9763215.5639 - (cdouble + csmallint)),
+ (-((-(cdouble))))
+FROM alltypesorc
+WHERE (((cstring2 <= '10')
+ OR ((ctinyint > cdouble)
+ AND (-5638.15 >= ctinyint)))
+ OR ((cdouble > 6981)
+ AND ((csmallint = 9763215.5639)
+ OR (cstring1 LIKE '%a'))))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cdouble,
+ ctimestamp1,
+ ctinyint,
+ cboolean1,
+ cstring1,
+ (-(cdouble)),
+ (cdouble + csmallint),
+ ((cdouble + csmallint) % 33),
+ (-(cdouble)),
+ (ctinyint % cdouble),
+ (ctinyint % csmallint),
+ (-(cdouble)),
+ (cbigint * (ctinyint % csmallint)),
+ (9763215.5639 - (cdouble + csmallint)),
+ (-((-(cdouble))))
+FROM alltypesorc
+WHERE (((cstring2 <= '10')
+ OR ((ctinyint > cdouble)
+ AND (-5638.15 >= ctinyint)))
+ OR ((cdouble > 6981)
+ AND ((csmallint = 9763215.5639)
+ OR (cstring1 LIKE '%a'))))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterStringGroupColLessEqualStringScalar(col 7, val 10) -> boolean, FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 12, col 5)(children: CastLongToDouble(col 0) -> 12:double) -> boolean, FilterDecimalScalarGreaterEqualDecimalColumn(val -5638.15, col 13)(children: CastLongToDecimal(col 0) -> 13:decimal(6,2)) -> boolean) -> boolean, FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 5, val 6981.0) -> boolean, FilterExprOrExpr(children: FilterDecimalColEqualDecimalScalar(col 14, val 9763215.5639)(children: CastLongToDecimal(col 1) -> 14:decimal(11,4)) -> boolean, FilterStringColLikeStringScalar(col 6, pattern %a) -> boolean) -> boolean) -> boolean) -> boolean
+ predicate: ((cstring2 <= '10') or ((UDFToDouble(ctinyint) > cdouble) and (-5638.15 >= CAST( ctinyint AS decimal(6,2)))) or ((cdouble > 6981.0) and ((CAST( csmallint AS decimal(11,4)) = 9763215.5639) or (cstring1 like '%a')))) (type: boolean)
+ Statistics: Num rows: 5461 Data size: 167650 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdouble (type: double), ctimestamp1 (type: timestamp), ctinyint (type: tinyint), cboolean1 (type: boolean), cstring1 (type: string), (- cdouble) (type: double), (cdouble + UDFToDouble(csmallint)) (type: double), ((cdouble + UDFToDouble(csmallint)) % 33.0) (type: double), (- cdouble) (type: double), (UDFToDouble(ctinyint) % cdouble) (type: double), (UDFToShort(ctinyint) % csmallint) (type: smallint), (- cdouble) (type: double), (cbigint * UDFToLong((UDFToShort(ctinyint) % csmallint))) (type: bigint), (9763215.5639 - (cdouble + UDFToDouble(csmallint))) (type: double), (- (- cdouble)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [5, 8, 0, 10, 6, 12, 16, 15, 17, 19, 20, 18, 22, 23, 25]
+ selectExpressions: DoubleColUnaryMinus(col 5) -> 12:double, DoubleColAddDoubleColumn(col 5, col 15)(children: CastLongToDouble(col 1) -> 15:double) -> 16:double, DoubleColModuloDoubleScalar(col 17, val 33.0)(children: DoubleColAddDoubleColumn(col 5, col 15)(children: CastLongToDouble(col 1) -> 15:double) -> 17:double) -> 15:double, DoubleColUnaryMinus(col 5) -> 17:double, DoubleColModuloDoubleColumn(col 18, col 5)(children: CastLongToDouble(col 0) -> 18:double) -> 19:double, LongColModuloLongColumn(col 0, col 1)(children: col 0) -> 20:long, DoubleColUnaryMinus(col 5) -> 18:double, LongColMultiplyLongColumn(col 3, col 21)(children: col 21) -> 22:long, DoubleScalarSubtractDoubleColumn(val 9763215.5639, col 24)(children: DoubleColAddDoubleColumn(col 5, col 23)(children: CastLongToDouble(col 1) -> 23:double) -> 24:double) -> 23:double, DoubleColUnaryMinus(col 24)(children: DoubleColUnaryMinus(col 5) -> 24:double) -> 25:double
+ Statistics: Num rows: 5461 Data size: 167650 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 5461 Data size: 167650 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 3, 5, 6, 7, 8, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, decimal(6,2), decimal(11,4), double, double, double, double, double, bigint, bigint, bigint, double, double, double
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT cdouble,
ctimestamp1,
ctinyint,
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_11.q.out b/ql/src/test/results/clientpositive/spark/vectorization_11.q.out
index dff58da..f79c3a0 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_11.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_11.q.out
@@ -1,3 +1,102 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cstring1,
+ cboolean1,
+ cdouble,
+ ctimestamp1,
+ (-3728 * csmallint),
+ (cdouble - 9763215.5639),
+ (-(cdouble)),
+ ((-(cdouble)) + 6981),
+ (cdouble * -5638.15)
+FROM alltypesorc
+WHERE ((cstring2 = cstring1)
+ OR ((ctimestamp1 IS NULL)
+ AND (cstring1 LIKE '%a')))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cstring1,
+ cboolean1,
+ cdouble,
+ ctimestamp1,
+ (-3728 * csmallint),
+ (cdouble - 9763215.5639),
+ (-(cdouble)),
+ ((-(cdouble)) + 6981),
+ (cdouble * -5638.15)
+FROM alltypesorc
+WHERE ((cstring2 = cstring1)
+ OR ((ctimestamp1 IS NULL)
+ AND (cstring1 LIKE '%a')))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterStringGroupColEqualStringGroupColumn(col 7, col 6) -> boolean, FilterExprAndExpr(children: SelectColumnIsNull(col 8) -> boolean, FilterStringColLikeStringScalar(col 6, pattern %a) -> boolean) -> boolean) -> boolean
+ predicate: ((cstring2 = cstring1) or (ctimestamp1 is null and (cstring1 like '%a'))) (type: boolean)
+ Statistics: Num rows: 9216 Data size: 282927 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double), ctimestamp1 (type: timestamp), (-3728 * UDFToInteger(csmallint)) (type: int), (cdouble - 9763215.5639) (type: double), (- cdouble) (type: double), ((- cdouble) + 6981.0) (type: double), (cdouble * -5638.15) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [6, 10, 5, 8, 12, 13, 14, 16, 15]
+ selectExpressions: LongScalarMultiplyLongColumn(val -3728, col 1)(children: col 1) -> 12:long, DoubleColSubtractDoubleScalar(col 5, val 9763215.5639) -> 13:double, DoubleColUnaryMinus(col 5) -> 14:double, DoubleColAddDoubleScalar(col 15, val 6981.0)(children: DoubleColUnaryMinus(col 5) -> 15:double) -> 16:double, DoubleColMultiplyDoubleScalar(col 5, val -5638.15) -> 15:double
+ Statistics: Num rows: 9216 Data size: 282927 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 9216 Data size: 282927 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [1, 5, 6, 7, 8, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, double, double, double, double
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT cstring1,
cboolean1,
cdouble,
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_12.q.out b/ql/src/test/results/clientpositive/spark/vectorization_12.q.out
index 6a7f69c..c17043b 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_12.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_12.q.out
@@ -1,3 +1,242 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cbigint,
+ cboolean1,
+ cstring1,
+ ctimestamp1,
+ cdouble,
+ (-6432 * cdouble),
+ (-(cbigint)),
+ COUNT(cbigint),
+ (cbigint * COUNT(cbigint)),
+ STDDEV_SAMP(cbigint),
+ ((-6432 * cdouble) / -6432),
+ (-(((-6432 * cdouble) / -6432))),
+ AVG(cdouble),
+ (-((-6432 * cdouble))),
+ (-5638.15 + cbigint),
+ SUM(cbigint),
+ (AVG(cdouble) / (-6432 * cdouble)),
+ AVG(cdouble),
+ (-((-(((-6432 * cdouble) / -6432))))),
+ (((-6432 * cdouble) / -6432) + (-((-6432 * cdouble)))),
+ STDDEV_POP(cdouble)
+FROM alltypesorc
+WHERE (((ctimestamp1 IS NULL)
+ AND ((cboolean1 >= cboolean2)
+ OR (ctinyint != csmallint)))
+ AND ((cstring1 LIKE '%a')
+ OR ((cboolean2 <= 1)
+ AND (cbigint >= csmallint))))
+GROUP BY cbigint, cboolean1, cstring1, ctimestamp1, cdouble
+ORDER BY ctimestamp1, cdouble, cbigint, cstring1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cbigint,
+ cboolean1,
+ cstring1,
+ ctimestamp1,
+ cdouble,
+ (-6432 * cdouble),
+ (-(cbigint)),
+ COUNT(cbigint),
+ (cbigint * COUNT(cbigint)),
+ STDDEV_SAMP(cbigint),
+ ((-6432 * cdouble) / -6432),
+ (-(((-6432 * cdouble) / -6432))),
+ AVG(cdouble),
+ (-((-6432 * cdouble))),
+ (-5638.15 + cbigint),
+ SUM(cbigint),
+ (AVG(cdouble) / (-6432 * cdouble)),
+ AVG(cdouble),
+ (-((-(((-6432 * cdouble) / -6432))))),
+ (((-6432 * cdouble) / -6432) + (-((-6432 * cdouble)))),
+ STDDEV_POP(cdouble)
+FROM alltypesorc
+WHERE (((ctimestamp1 IS NULL)
+ AND ((cboolean1 >= cboolean2)
+ OR (ctinyint != csmallint)))
+ AND ((cstring1 LIKE '%a')
+ OR ((cboolean2 <= 1)
+ AND (cbigint >= csmallint))))
+GROUP BY cbigint, cboolean1, cstring1, ctimestamp1, cdouble
+ORDER BY ctimestamp1, cdouble, cbigint, cstring1
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (SORT, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 8) -> boolean, FilterExprOrExpr(children: FilterLongColGreaterEqualLongColumn(col 10, col 11) -> boolean, FilterLongColNotEqualLongColumn(col 0, col 1)(children: col 0) -> boolean) -> boolean, FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 6, pattern %a) -> boolean, FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 11, val 1) -> boolean, FilterLongColGreaterEqualLongColumn(col 3, col 1)(children: col 1) -> boolean) -> boolean) -> boolean) -> boolean
+ predicate: (ctimestamp1 is null and ((cboolean1 >= cboolean2) or (UDFToShort(ctinyint) <> csmallint)) and ((cstring1 like '%a') or ((cboolean2 <= 1) and (cbigint >= UDFToLong(csmallint))))) (type: boolean)
+ Statistics: Num rows: 5006 Data size: 153682 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbigint (type: bigint), cdouble (type: double), cstring1 (type: string), cboolean1 (type: boolean)
+ outputColumnNames: cbigint, cdouble, cstring1, cboolean1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [3, 5, 6, 10]
+ Statistics: Num rows: 5006 Data size: 153682 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(cbigint), stddev_samp(cbigint), avg(cdouble), sum(cbigint), stddev_pop(cdouble)
+ Group By Vectorization:
+ aggregators: VectorUDAFCount(col 3) -> bigint, VectorUDAFStdSampLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 5) -> struct<count:bigint,sum:double,input:double>, VectorUDAFSumLong(col 3) -> bigint, VectorUDAFStdPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double>
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 5, col 3, col 6, col 10
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4]
+ keys: cdouble (type: double), cbigint (type: bigint), cstring1 (type: string), cboolean1 (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 5006 Data size: 153682 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double), _col1 (type: bigint), _col2 (type: string), _col3 (type: boolean)
+ sort order: ++++
+ Map-reduce partition columns: _col0 (type: double), _col1 (type: bigint), _col2 (type: string), _col3 (type: boolean)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0, 1, 2, 3]
+ valueColumns: [4, 5, 6, 7, 8]
+ Statistics: Num rows: 5006 Data size: 153682 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col4 (type: bigint), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,input:double>), _col7 (type: bigint), _col8 (type: struct<count:bigint,sum:double,variance:double>)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 3, 5, 6, 8, 10, 11]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ reduceColumnNullOrder: aaaa
+ reduceColumnSortOrder: ++++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 9
+ dataColumns: KEY._col0:double, KEY._col1:bigint, KEY._col2:string, KEY._col3:boolean, VALUE._col0:bigint, VALUE._col1:struct<count:bigint,sum:double,variance:double>, VALUE._col2:struct<count:bigint,sum:double,input:double>, VALUE._col3:bigint, VALUE._col4:struct<count:bigint,sum:double,variance:double>
+ partitionColumnCount: 0
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), stddev_samp(VALUE._col1), avg(VALUE._col2), sum(VALUE._col3), stddev_pop(VALUE._col4)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 4) -> bigint, VectorUDAFStdSampFinal(col 5) -> double, VectorUDAFAvgFinal(col 6) -> double, VectorUDAFSumLong(col 7) -> bigint, VectorUDAFStdPopFinal(col 8) -> double
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ vectorOutput: true
+ keyExpressions: col 0, col 1, col 2, col 3
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumns: [0, 1, 2, 3, 4]
+ keys: KEY._col0 (type: double), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: bigint), _col3 (type: boolean), _col2 (type: string), _col0 (type: double), (-6432.0 * _col0) (type: double), (- _col1) (type: bigint), _col4 (type: bigint), (_col1 * _col4) (type: bigint), _col5 (type: double), ((-6432.0 * _col0) / -6432.0) (type: double), (- ((-6432.0 * _col0) / -6432.0)) (type: double), _col6 (type: double), (- (-6432.0 * _col0)) (type: double), (-5638.15 + CAST( _col1 AS decimal(19,0))) (type: decimal(22,2)), _col7 (type: bigint), (_col6 / (-6432.0 * _col0)) (type: double), (- (- ((-6432.0 * _col0) / -6432.0))) (type: double), (((-6432.0 * _col0) / -6432.0) + (- (-6432.0 * _col0))) (type: double), _col8 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col17, _col18, _col19
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [1, 3, 2, 0, 9, 10, 4, 11, 5, 13, 12, 6, 15, 17, 7, 18, 19, 14, 8]
+ selectExpressions: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 9:double, LongColUnaryMinus(col 1) -> 10:long, LongColMultiplyLongColumn(col 1, col 4) -> 11:long, DoubleColDivideDoubleScalar(col 12, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 12:double) -> 13:double, DoubleColUnaryMinus(col 14)(children: DoubleColDivideDoubleScalar(col 12, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 12:double) -> 14:double) -> 12:double, DoubleColUnaryMinus(col 14)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 14:double) -> 15:double, DecimalScalarAddDecimalColumn(val -5638.15, col 16)(children: CastLongToDecimal(col 1) -> 16:decimal(19,0)) -> 17:decimal(22,2), DoubleColDivideDoubleColumn(col 6, col 14)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 14:double) -> 18:double, DoubleColUnaryMinus(col 14)(children: DoubleColUnaryMinus(col 19)(children: DoubleColDi
videDoubleScalar(col 14, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 14:double) -> 19:double) -> 14:double) -> 19:double, DoubleColAddDoubleColumn(col 20, col 21)(children: DoubleColDivideDoubleScalar(col 14, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 14:double) -> 20:double, DoubleColUnaryMinus(col 14)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0) -> 14:double) -> 21:double) -> 14:double
+ Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: double), _col0 (type: bigint), _col2 (type: string)
+ sort order: +++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: [3, 9, 10, 4, 11, 5, 13, 12, 6, 15, 17, 7, 18, 19, 14, 8]
+ Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: boolean), _col4 (type: double), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(22,2)), _col14 (type: bigint), _col15 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double)
+ Reducer 3
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ reduceColumnNullOrder: aaa
+ reduceColumnSortOrder: +++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 19
+ dataColumns: KEY.reducesinkkey0:double, KEY.reducesinkkey1:bigint, KEY.reducesinkkey2:string, VALUE._col0:boolean, VALUE._col1:double, VALUE._col2:bigint, VALUE._col3:bigint, VALUE._col4:bigint, VALUE._col5:double, VALUE._col6:double, VALUE._col7:double, VALUE._col8:double, VALUE._col9:double, VALUE._col10:decimal(22,2), VALUE._col11:bigint, VALUE._col12:double, VALUE._col13:double, VALUE._col14:double, VALUE._col15:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: timestamp
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: boolean), KEY.reducesinkkey2 (type: string), null (type: timestamp), KEY.reducesinkkey0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: bigint), VALUE._col3 (type: bigint), VALUE._col4 (type: bigint), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: decimal(22,2)), VALUE._col11 (type: bigint), VALUE._col12 (type: double), VALUE._col8 (type: double), VALUE._col13 (type: double), VALUE._col14 (type: double), VALUE._col15 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [1, 3, 2, 19, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 11, 16, 17, 18]
+ selectExpressions: ConstantVectorExpression(val null) -> 19:timestamp
+ Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT cbigint,
cboolean1,
cstring1,
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
index 9a4c6c3..bae304b 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cboolean1,
ctinyint,
ctimestamp1,
@@ -31,7 +31,7 @@ GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16
LIMIT 40
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cboolean1,
ctinyint,
ctimestamp1,
@@ -108,11 +108,12 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFMaxLong(col 0) -> tinyint, VectorUDAFSumDouble(col 4) -> double, VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 4) -> float, VectorUDAFMinLong(col 0) -> tinyint
className: VectorGroupByOperator
- vectorOutput: false
+ groupByMode: HASH
+ vectorOutput: true
keyExpressions: col 10, col 0, col 8, col 4, col 6
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1, 2, 3, 4, 5]
- vectorOutputConditionsNotMet: Vector output of VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false
keys: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -121,26 +122,57 @@ STAGE PLANS:
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
sort order: +++++
Map-reduce partition columns: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3, 4]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0, 1, 2, 3, 4]
+ valueColumns: [5, 6, 7, 8, 9, 10]
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>), _col9 (type: float), _col10 (type: tinyint)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 4, 5, 6, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, decimal(11,4)
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: Aggregation Function UDF stddev_pop parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col2] not supported
- vectorized: false
+ reduceColumnNullOrder: aaaaa
+ reduceColumnSortOrder: +++++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 11
+ dataColumns: KEY._col0:boolean, KEY._col1:tinyint, KEY._col2:timestamp, KEY._col3:float, KEY._col4:string, VALUE._col0:tinyint, VALUE._col1:double, VALUE._col2:struct<count:bigint,sum:double,variance:double>, VALUE._col3:struct<count:bigint,sum:double,variance:double>, VALUE._col4:float, VALUE._col5:tinyint
+ partitionColumnCount: 0
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), min(VALUE._col5)
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 5) -> tinyint, VectorUDAFSumDouble(col 6) -> double, VectorUDAFStdPopFinal(col 7) -> double, VectorUDAFStdPopFinal(col 8) -> double, VectorUDAFMaxDouble(col 9) -> float, VectorUDAFMinLong(col 10) -> tinyint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ vectorOutput: true
+ keyExpressions: col 0, col 1, col 2, col 3, col 4
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
keys: KEY._col0 (type: boolean), KEY._col1 (type: tinyint), KEY._col2 (type: timestamp), KEY._col3 (type: float), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -148,10 +180,21 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: float), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (CAST( ((- _col1) + _col5) AS decimal(3,0)) - 10.175) (type: decimal(7,3)), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col10 (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 11, 5, 13, 6, 16, 15, 17, 7, 18, 8, 20, 22, 21, 9, 25, 10]
+ selectExpressions: LongColUnaryMinus(col 1) -> 11:long, LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 13:long, DoubleColMultiplyDoubleColumn(col 6, col 15)(children: CastLongToDouble(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 15:double) -> 16:double, DoubleColUnaryMinus(col 6) -> 15:double, DoubleScalarMultiplyDoubleColumn(val 79.5530014038086, col 3) -> 17:double, DoubleColUnaryMinus(col 6) -> 18:double, DecimalColSubtractDecimalScalar(col 19, val 10.175)(children: CastLongToDecimal(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 19:decimal(3,0)) -> 20:decimal(7,3), DoubleColUnaryMinus(col 21)(children: DoubleColUnaryMinus(col 6) -> 21:double) -> 22:double, DoubleScalarDivideDoubleColumn(val -26.28, col 23)(children: DoubleColUnaryMinus(col 21)(children: DoubleColUnaryMinus(col 6) -
> 21:double) -> 23:double) -> 21:double, DoubleColDivideDoubleColumn(col 24, col 23)(children: DoubleColMultiplyDoubleColumn(col 6, col 23)(children: CastLongToDouble(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 23:double) -> 24:double, CastLongToDouble(col 1) -> 23:double) -> 25:double
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: decimal(7,3)), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
sort order: +++++++++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3, 4, 11, 5, 13, 6, 16, 15, 17, 7, 18, 8, 20, 22, 21, 9, 25, 10]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: []
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reducer 3
@@ -159,10 +202,16 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ reduceColumnNullOrder: aaaaaaaaaaaaaaaaaaaaa
+ reduceColumnSortOrder: +++++++++++++++++++++
groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 21
+ dataColumns: KEY.reducesinkkey0:boolean, KEY.reducesinkkey1:tinyint, KEY.reducesinkkey2:timestamp, KEY.reducesinkkey3:float, KEY.reducesinkkey4:string, KEY.reducesinkkey5:tinyint, KEY.reducesinkkey6:tinyint, KEY.reducesinkkey7:tinyint, KEY.reducesinkkey8:double, KEY.reducesinkkey9:double, KEY.reducesinkkey10:double, KEY.reducesinkkey11:float, KEY.reducesinkkey12:double, KEY.reducesinkkey13:double, KEY.reducesinkkey14:double, KEY.reducesinkkey15:decimal(7,3), KEY.reducesinkkey16:double, KEY.reducesinkkey17:double, KEY.reducesinkkey18:float, KEY.reducesinkkey19:double, KEY.reducesinkkey20:tinyint
+ partitionColumnCount: 0
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: tinyint), KEY.reducesinkkey6 (type: tinyint), KEY.reducesinkkey7 (type: tinyint), KEY.reducesinkkey8 (type: double), KEY.reducesinkkey9 (type: double), KEY.reducesinkkey10 (type: double), KEY.reducesinkkey11 (type: float), KEY.reducesinkkey12 (type: double), KEY.reducesinkkey10 (type: double), KEY.reducesinkkey14 (type: double), KEY.reducesinkkey15 (type: decimal(7,3)), KEY.reducesinkkey16 (type: double), KEY.reducesinkkey17 (type: double), KEY.reducesinkkey18 (type: float), KEY.reducesinkkey19 (type: double), KEY.reducesinkkey20 (type: tinyint)
@@ -413,11 +462,12 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFMaxLong(col 0) -> tinyint, VectorUDAFSumDouble(col 4) -> double, VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 4) -> float, VectorUDAFMinLong(col 0) -> tinyint
className: VectorGroupByOperator
- vectorOutput: false
+ groupByMode: HASH
+ vectorOutput: true
keyExpressions: col 10, col 0, col 8, col 4, col 6
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1, 2, 3, 4, 5]
- vectorOutputConditionsNotMet: Vector output of VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false
keys: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -426,26 +476,42 @@ STAGE PLANS:
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
sort order: +++++
Map-reduce partition columns: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>), _col9 (type: float), _col10 (type: tinyint)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: Aggregation Function UDF stddev_pop parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col2] not supported
- vectorized: false
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), min(VALUE._col5)
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 5) -> tinyint, VectorUDAFSumDouble(col 6) -> double, VectorUDAFStdPopFinal(col 7) -> double, VectorUDAFStdPopFinal(col 8) -> double, VectorUDAFMaxDouble(col 9) -> float, VectorUDAFMinLong(col 10) -> tinyint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ vectorOutput: true
+ keyExpressions: col 0, col 1, col 2, col 3, col 4
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
keys: KEY._col0 (type: boolean), KEY._col1 (type: tinyint), KEY._col2 (type: timestamp), KEY._col3 (type: float), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -453,10 +519,19 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: float), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (CAST( ((- _col1) + _col5) AS decimal(3,0)) - 10.175) (type: decimal(7,3)), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col10 (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 11, 5, 13, 6, 16, 15, 17, 7, 18, 8, 20, 22, 21, 9, 25, 10]
+ selectExpressions: LongColUnaryMinus(col 1) -> 11:long, LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 13:long, DoubleColMultiplyDoubleColumn(col 6, col 15)(children: CastLongToDouble(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 15:double) -> 16:double, DoubleColUnaryMinus(col 6) -> 15:double, DoubleScalarMultiplyDoubleColumn(val 79.5530014038086, col 3) -> 17:double, DoubleColUnaryMinus(col 6) -> 18:double, DecimalColSubtractDecimalScalar(col 19, val 10.175)(children: CastLongToDecimal(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 19:decimal(3,0)) -> 20:decimal(7,3), DoubleColUnaryMinus(col 21)(children: DoubleColUnaryMinus(col 6) -> 21:double) -> 22:double, DoubleScalarDivideDoubleColumn(val -26.28, col 23)(children: DoubleColUnaryMinus(col 21)(children: DoubleColUnaryMinus(col 6) -
> 21:double) -> 23:double) -> 21:double, DoubleColDivideDoubleColumn(col 24, col 23)(children: DoubleColMultiplyDoubleColumn(col 6, col 23)(children: CastLongToDouble(col 14)(children: LongColAddLongColumn(col 12, col 5)(children: LongColUnaryMinus(col 1) -> 12:long) -> 14:long) -> 23:double) -> 24:double, CastLongToDouble(col 1) -> 23:double) -> 25:double
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: decimal(7,3)), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
sort order: +++++++++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reducer 3
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_14.q.out b/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
index 1541908..9d52abe 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT ctimestamp1,
cfloat,
cstring1,
@@ -31,7 +31,7 @@ WHERE (((ctinyint <= cbigint)
GROUP BY ctimestamp1, cfloat, cstring1, cboolean1, cdouble
ORDER BY cstring1, cfloat, cdouble, ctimestamp1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT ctimestamp1,
cfloat,
cstring1,
@@ -85,15 +85,36 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 0, col 3)(children: col 0) -> boolean, FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleColumn(col 12, col 5)(children: CastLongToDouble(col 2) -> 12:double) -> boolean, FilterTimestampColLessTimestampColumn(col 9, col 8) -> boolean) -> boolean, FilterDoubleColLessDoubleColumn(col 5, col 12)(children: CastLongToDouble(col 0) -> 12:double) -> boolean, FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 3, val -257) -> boolean, FilterDoubleColLessDoubleColumn(col 4, col 12)(children: CastLongToFloatViaLongToDouble(col 2) -> 12:double) -> boolean) -> boolean) -> boolean
predicate: ((UDFToLong(ctinyint) <= cbigint) and ((UDFToDouble(cint) <= cdouble) or (ctimestamp2 < ctimestamp1)) and (cdouble < UDFToDouble(ctinyint)) and ((cbigint > -257) or (cfloat < UDFToFloat(cint)))) (type: boolean)
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double), (- (-26.28 + cdouble)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [8, 4, 6, 10, 5, 13]
+ selectExpressions: DoubleColUnaryMinus(col 12)(children: DoubleScalarAddDoubleColumn(val -26.28, col 5) -> 12:double) -> 13:double
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: stddev_samp(_col5), max(_col1), stddev_pop(_col1), count(_col1), var_pop(_col1), var_samp(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFStdSampDouble(col 13) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 4) -> float, VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCount(col 4) -> bigint, VectorUDAFVarPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double>
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 6, col 4, col 5, col 8, col 10
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
keys: _col2 (type: string), _col1 (type: float), _col4 (type: double), _col0 (type: timestamp), _col3 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -102,26 +123,57 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: float), _col2 (type: double), _col3 (type: timestamp), _col4 (type: boolean)
sort order: +++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: float), _col2 (type: double), _col3 (type: timestamp), _col4 (type: boolean)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3, 4]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0, 1, 2, 3, 4]
+ valueColumns: [5, 6, 7, 8, 9, 10]
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: float), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: bigint), _col9 (type: struct<count:bigint,sum:double,variance:double>), _col10 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 2, 3, 4, 5, 6, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, double
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: Aggregation Function UDF stddev_samp parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported
- vectorized: false
+ reduceColumnNullOrder: aaaaa
+ reduceColumnSortOrder: +++++
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 11
+ dataColumns: KEY._col0:string, KEY._col1:float, KEY._col2:double, KEY._col3:timestamp, KEY._col4:boolean, VALUE._col0:struct<count:bigint,sum:double,variance:double>, VALUE._col1:float, VALUE._col2:struct<count:bigint,sum:double,variance:double>, VALUE._col3:bigint, VALUE._col4:struct<count:bigint,sum:double,variance:double>, VALUE._col5:struct<count:bigint,sum:double,variance:double>
+ partitionColumnCount: 0
Reduce Operator Tree:
Group By Operator
aggregations: stddev_samp(VALUE._col0), max(VALUE._col1), stddev_pop(VALUE._col2), count(VALUE._col3), var_pop(VALUE._col4), var_samp(VALUE._col5)
+ Group By Vectorization:
+ aggregators: VectorUDAFStdSampFinal(col 5) -> double, VectorUDAFMaxDouble(col 6) -> float, VectorUDAFStdPopFinal(col 7) -> double, VectorUDAFCountMerge(col 8) -> bigint, VectorUDAFVarPopFinal(col 9) -> double, VectorUDAFVarSampFinal(col 10) -> double
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ vectorOutput: true
+ keyExpressions: col 0, col 1, col 2, col 3, col 4
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
keys: KEY._col0 (type: string), KEY._col1 (type: float), KEY._col2 (type: double), KEY._col3 (type: timestamp), KEY._col4 (type: boolean)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
@@ -129,10 +181,21 @@ STAGE PLANS:
Select Operator
expressions: _col3 (type: timestamp), _col1 (type: float), _col0 (type: string), _col4 (type: boolean), _col2 (type: double), (-26.28 + _col2) (type: double), (- (-26.28 + _col2)) (type: double), _col5 (type: double), (_col1 * -26.28) (type: float), _col6 (type: float), (- _col1) (type: float), (- _col6) (type: float), ((- (-26.28 + _col2)) / 10.175) (type: double), _col7 (type: double), _col8 (type: bigint), (- ((- (-26.28 + _col2)) / 10.175)) (type: double), (-1.389 % _col5) (type: double), (UDFToDouble(_col1) - _col2) (type: double), _col9 (type: double), (_col9 % 10.175) (type: double), _col10 (type: double), (- (UDFToDouble(_col1) - _col2)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [3, 1, 0, 4, 2, 11, 13, 5, 12, 6, 14, 15, 16, 7, 8, 18, 17, 19, 9, 20, 10, 22]
+ selectExpressions: DoubleScalarAddDoubleColumn(val -26.28, col 2) -> 11:double, DoubleColUnaryMinus(col 12)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2) -> 12:double) -> 13:double, DoubleColMultiplyDoubleScalar(col 1, val -26.280000686645508) -> 12:double, DoubleColUnaryMinus(col 1) -> 14:double, DoubleColUnaryMinus(col 6) -> 15:double, DoubleColDivideDoubleScalar(col 17, val 10.175)(children: DoubleColUnaryMinus(col 16)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2) -> 16:double) -> 17:double) -> 16:double, DoubleColUnaryMinus(col 17)(children: DoubleColDivideDoubleScalar(col 18, val 10.175)(children: DoubleColUnaryMinus(col 17)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2) -> 17:double) -> 18:double) -> 17:double) -> 18:double, DoubleScalarModuloDoubleColumn(val -1.389, col 5) -> 17:double, DoubleColSubtractDoubleColumn(col 1, col 2)(children: col 1) -> 19:double, DoubleColModuloDoubleScalar(col 9, val 10.175) -> 20:double, Do
ubleColUnaryMinus(col 21)(children: DoubleColSubtractDoubleColumn(col 1, col 2)(children: col 1) -> 21:double) -> 22:double
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: string), _col1 (type: float), _col4 (type: double), _col0 (type: timestamp)
sort order: ++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: [4, 11, 13, 5, 12, 6, 14, 15, 16, 7, 8, 18, 17, 19, 9, 20, 10, 22]
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: boolean), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: float), _col10 (type: float), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: bigint), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double), _col20 (type: double), _col21 (type: double)
Reducer 3
@@ -140,17 +203,30 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ reduceColumnNullOrder: aaaa
+ reduceColumnSortOrder: ++++
groupByVectorOutput: true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 22
+ dataColumns: KEY.reducesinkkey0:string, KEY.reducesinkkey1:float, KEY.reducesinkkey2:double, KEY.reducesinkkey3:timestamp, VALUE._col0:boolean, VALUE._col1:double, VALUE._col2:double, VALUE._col3:double, VALUE._col4:float, VALUE._col5:float, VALUE._col6:float, VALUE._col7:float, VALUE._col8:double, VALUE._col9:double, VALUE._col10:bigint, VALUE._col11:double, VALUE._col12:double, VALUE._col13:double, VALUE._col14:double, VALUE._col15:double, VALUE._col16:double, VALUE._col17:double
+ partitionColumnCount: 0
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey3 (type: timestamp), KEY.reducesinkkey1 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col0 (type: boolean), KEY.reducesinkkey2 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: float), VALUE._col6 (type: float), VALUE._col7 (type: float), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: bigint), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col14 (type: double), VALUE._col15 (type: double), VALUE._col16 (type: double), VALUE._col17 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [3, 1, 0, 4, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
index b2d8aaf..cc9ae1d 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cfloat,
cboolean1,
cdouble,
@@ -29,7 +29,7 @@ WHERE (((cstring2 LIKE '%ss%')
GROUP BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
ORDER BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cfloat,
cboolean1,
cdouble,
@@ -81,15 +81,35 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7, pattern %ss%) -> boolean, FilterStringColLikeStringScalar(col 6, pattern 10%) -> boolean, FilterExprAndExpr(children: FilterLongColGreaterEqualLongScalar(col 2, val -75) -> boolean, FilterLongColEqualLongColumn(col 0, col 1)(children: col 0) -> boolean, FilterDoubleColGreaterEqualDoubleScalar(col 5, val -3728.0) -> boolean) -> boolean) -> boolean
predicate: ((cstring2 like '%ss%') or (cstring1 like '10%') or ((cint >= -75) and (UDFToShort(ctinyint) = csmallint) and (cdouble >= -3728.0))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int), cfloat (type: float), cdouble (type: double), cstring1 (type: string), ctimestamp1 (type: timestamp), cboolean1 (type: boolean)
outputColumnNames: ctinyint, cint, cfloat, cdouble, cstring1, ctimestamp1, cboolean1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 2, 4, 5, 6, 8, 10]
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: stddev_samp(cfloat), min(cdouble), stddev_samp(ctinyint), var_pop(ctinyint), var_samp(cint), stddev_pop(cint)
+ Group By Vectorization:
+ aggregators: VectorUDAFStdSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMinDouble(col 5) -> double, VectorUDAFStdSampLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 4, col 10, col 5, col 6, col 0, col 2, col 8
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
keys: cfloat (type: float), cboolean1 (type: boolean), cdouble (type: double), cstring1 (type: string), ctinyint (type: tinyint), cint (type: int), ctimestamp1 (type: timestamp)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
@@ -98,26 +118,43 @@ STAGE PLANS:
key expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp)
sort order: +++++++
Map-reduce partition columns: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: [0, 1, 2, 3, 4, 5, 6]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: [0, 1, 2, 3, 4, 5, 6]
+ valueColumns: [7, 8, 9, 10, 11, 12]
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: double), _col9 (type: struct<count:bigint,sum:double,variance:double>), _col10 (type: struct<count:bigint,sum:double,variance:double>), _col11 (type: struct<count:bigint,sum:double,variance:double>), _col12 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 4, 5, 6, 7, 8, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
Reducer 2
Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: Aggregation Function UDF stddev_samp parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported
- vectorized: false
+ enabled: false
+ enableConditionsMet: hive.execution.engine spark IN [tez, spark] IS true
+ enableConditionsNotMet: hive.vectorized.execution.reduce.enabled IS false
Reduce Operator Tree:
Group By Operator
aggregations: stddev_samp(VALUE._col0), min(VALUE._col1), stddev_samp(VALUE._col2), var_pop(VALUE._col3), var_samp(VALUE._col4), stddev_pop(VALUE._col5)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
keys: KEY._col0 (type: float), KEY._col1 (type: boolean), KEY._col2 (type: double), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int), KEY._col6 (type: timestamp)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
@@ -132,14 +169,10 @@ STAGE PLANS:
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: double), _col8 (type: decimal(13,2)), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: tinyint), _col16 (type: double), _col17 (type: float), _col18 (type: int), _col19 (type: decimal(13,2)), _col20 (type: double)
Reducer 3
- Execution mode: vectorized
Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- groupByVectorOutput: true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ enabled: false
+ enableConditionsMet: hive.execution.engine spark IN [tez, spark] IS true
+ enableConditionsNotMet: hive.vectorized.execution.reduce.enabled IS false
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: boolean), KEY.reducesinkkey2 (type: double), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: tinyint), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: timestamp), VALUE._col0 (type: double), VALUE._col1 (type: decimal(13,2)), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: tinyint), VALUE._col9 (type: double), VALUE._col10 (type: float), VALUE._col11 (type: int), VALUE._col12 (type: decimal(13,2)), VALUE._col13 (type: double)