You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/06/22 23:40:38 UTC
[02/34] hive git commit: HIVE-16589: Vectorization: Support Complex
Types and GroupBy modes PARTIAL2, FINAL, and COMPLETE for AVG,
VARIANCE (Matt McCline, reviewed by Jason Dere)
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_3.q.out b/ql/src/test/results/clientpositive/vectorization_3.q.out
index 2398dee..a730ca6 100644
--- a/ql/src/test/results/clientpositive/vectorization_3.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_3.q.out
@@ -1,4 +1,159 @@
WARNING: Comparing a bigint and a double may result in a loss of precision.
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT STDDEV_SAMP(csmallint),
+ (STDDEV_SAMP(csmallint) - 10.175),
+ STDDEV_POP(ctinyint),
+ (STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)),
+ (-(STDDEV_POP(ctinyint))),
+ (STDDEV_SAMP(csmallint) % 79.553),
+ (-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))),
+ STDDEV_SAMP(cfloat),
+ (-(STDDEV_SAMP(csmallint))),
+ SUM(cfloat),
+ ((-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))) / (STDDEV_SAMP(csmallint) - 10.175)),
+ (-((STDDEV_SAMP(csmallint) - 10.175))),
+ AVG(cint),
+ (-3728 - STDDEV_SAMP(csmallint)),
+ STDDEV_POP(cint),
+ (AVG(cint) / STDDEV_SAMP(cfloat))
+FROM alltypesorc
+WHERE (((cint <= cfloat)
+ AND ((79.553 != cbigint)
+ AND (ctimestamp2 = -29071)))
+ OR ((cbigint > cdouble)
+ AND ((79.553 <= csmallint)
+ AND (ctimestamp1 > ctimestamp2))))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT STDDEV_SAMP(csmallint),
+ (STDDEV_SAMP(csmallint) - 10.175),
+ STDDEV_POP(ctinyint),
+ (STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)),
+ (-(STDDEV_POP(ctinyint))),
+ (STDDEV_SAMP(csmallint) % 79.553),
+ (-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))),
+ STDDEV_SAMP(cfloat),
+ (-(STDDEV_SAMP(csmallint))),
+ SUM(cfloat),
+ ((-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))) / (STDDEV_SAMP(csmallint) - 10.175)),
+ (-((STDDEV_SAMP(csmallint) - 10.175))),
+ AVG(cint),
+ (-3728 - STDDEV_SAMP(csmallint)),
+ STDDEV_POP(cint),
+ (AVG(cint) / STDDEV_SAMP(cfloat))
+FROM alltypesorc
+WHERE (((cint <= cfloat)
+ AND ((79.553 != cbigint)
+ AND (ctimestamp2 = -29071)))
+ OR ((cbigint > cdouble)
+ AND ((79.553 <= csmallint)
+ AND (ctimestamp1 > ctimestamp2))))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleColumn(col 12, col 4)(children: CastLongToFloatViaLongToDouble(col 2) -> 12:double) -> boolean, FilterDecimalScalarNotEqualDecimalColumn(val 79.553, col 13)(children: CastLongToDecimal(col 3) -> 13:decimal(22,3)) -> boolean, FilterDoubleColEqualDoubleScalar(col 12, val -29071.0)(children: CastTimestampToDouble(col 9) -> 12:double) -> boolean) -> boolean, FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 12, col 5)(children: CastLongToDouble(col 3) -> 12:double) -> boolean, FilterDecimalScalarLessEqualDecimalColumn(val 79.553, col 14)(children: CastLongToDecimal(col 1) -> 14:decimal(8,3)) -> boolean, FilterTimestampColGreaterTimestampColumn(col 8, col 9) -> boolean) -> boolean) -> boolean
+ predicate: (((UDFToFloat(cint) <= cfloat) and (79.553 <> CAST( cbigint AS decimal(22,3))) and (UDFToDouble(ctimestamp2) = -29071.0)) or ((UDFToDouble(cbigint) > cdouble) and (79.553 <= CAST( csmallint AS decimal(8,3))) and (ctimestamp1 > ctimestamp2))) (type: boolean)
+ Statistics: Num rows: 2503 Data size: 538153 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cfloat (type: float)
+ outputColumnNames: ctinyint, csmallint, cint, cfloat
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2, 4]
+ Statistics: Num rows: 2503 Data size: 538153 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: stddev_samp(csmallint), stddev_pop(ctinyint), stddev_samp(cfloat), sum(cfloat), avg(cint), stddev_pop(cint)
+ Group By Vectorization:
+ aggregators: VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumDouble(col 4) -> double, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5]
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: struct<count:bigint,sum:double,variance:double>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,input:int>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 4, 5, 8, 9]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, decimal(22,3), decimal(8,3)
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: stddev_samp(VALUE._col0), stddev_pop(VALUE._col1), stddev_samp(VALUE._col2), sum(VALUE._col3), avg(VALUE._col4), stddev_pop(VALUE._col5)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: double), (_col0 - 10.175) (type: double), _col1 (type: double), (_col0 * (_col0 - 10.175)) (type: double), (- _col1) (type: double), (_col0 % 79.553) (type: double), (- (_col0 * (_col0 - 10.175))) (type: double), _col2 (type: double), (- _col0) (type: double), _col3 (type: double), ((- (_col0 * (_col0 - 10.175))) / (_col0 - 10.175)) (type: double), (- (_col0 - 10.175)) (type: double), _col4 (type: double), (-3728.0 - _col0) (type: double), _col5 (type: double), (_col4 / _col2) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+WARNING: Comparing a bigint and a double may result in a loss of precision.
PREHOOK: query: SELECT STDDEV_SAMP(csmallint),
(STDDEV_SAMP(csmallint) - 10.175),
STDDEV_POP(ctinyint),
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_4.q.out b/ql/src/test/results/clientpositive/vectorization_4.q.out
index 0d6829f..0199d7d 100644
--- a/ql/src/test/results/clientpositive/vectorization_4.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_4.q.out
@@ -1,3 +1,152 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT SUM(cint),
+ (SUM(cint) * -563),
+ (-3728 + SUM(cint)),
+ STDDEV_POP(cdouble),
+ (-(STDDEV_POP(cdouble))),
+ AVG(cdouble),
+ ((SUM(cint) * -563) % SUM(cint)),
+ (((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)),
+ VAR_POP(cdouble),
+ (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))),
+ ((-3728 + SUM(cint)) - (SUM(cint) * -563)),
+ MIN(ctinyint),
+ MIN(ctinyint),
+ (MIN(ctinyint) * (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))))
+FROM alltypesorc
+WHERE (((csmallint >= cint)
+ OR ((-89010 >= ctinyint)
+ AND (cdouble > 79.553)))
+ OR ((-563 != cbigint)
+ AND ((ctinyint != cbigint)
+ OR (-3728 >= cdouble))))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT SUM(cint),
+ (SUM(cint) * -563),
+ (-3728 + SUM(cint)),
+ STDDEV_POP(cdouble),
+ (-(STDDEV_POP(cdouble))),
+ AVG(cdouble),
+ ((SUM(cint) * -563) % SUM(cint)),
+ (((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)),
+ VAR_POP(cdouble),
+ (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))),
+ ((-3728 + SUM(cint)) - (SUM(cint) * -563)),
+ MIN(ctinyint),
+ MIN(ctinyint),
+ (MIN(ctinyint) * (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))))
+FROM alltypesorc
+WHERE (((csmallint >= cint)
+ OR ((-89010 >= ctinyint)
+ AND (cdouble > 79.553)))
+ OR ((-563 != cbigint)
+ AND ((ctinyint != cbigint)
+ OR (-3728 >= cdouble))))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterEqualLongColumn(col 1, col 2)(children: col 1) -> boolean, FilterExprAndExpr(children: FilterLongScalarGreaterEqualLongColumn(val -89010, col 0)(children: col 0) -> boolean, FilterDoubleColGreaterDoubleScalar(col 5, val 79.553) -> boolean) -> boolean, FilterExprAndExpr(children: FilterLongScalarNotEqualLongColumn(val -563, col 3) -> boolean, FilterExprOrExpr(children: FilterLongColNotEqualLongColumn(col 0, col 3)(children: col 0) -> boolean, FilterDoubleScalarGreaterEqualDoubleColumn(val -3728.0, col 5) -> boolean) -> boolean) -> boolean) -> boolean
+ predicate: ((UDFToInteger(csmallint) >= cint) or ((-89010 >= UDFToInteger(ctinyint)) and (cdouble > 79.553)) or ((-563 <> cbigint) and ((UDFToLong(ctinyint) <> cbigint) or (-3728.0 >= cdouble)))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cint (type: int), cdouble (type: double)
+ outputColumnNames: ctinyint, cint, cdouble
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 2, 5]
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(cint), stddev_pop(cdouble), avg(cdouble), var_pop(cdouble), min(ctinyint)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 2) -> bigint, VectorUDAFStdPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 5) -> struct<count:bigint,sum:double,input:double>, VectorUDAFVarPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMinLong(col 0) -> tinyint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4]
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,input:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: tinyint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 5]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), stddev_pop(VALUE._col1), avg(VALUE._col2), var_pop(VALUE._col3), min(VALUE._col4)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint), (_col0 * -563) (type: bigint), (-3728 + _col0) (type: bigint), _col1 (type: double), (- _col1) (type: double), _col2 (type: double), ((_col0 * -563) % _col0) (type: bigint), (UDFToDouble(((_col0 * -563) % _col0)) / _col2) (type: double), _col3 (type: double), (- (UDFToDouble(((_col0 * -563) % _col0)) / _col2)) (type: double), ((-3728 + _col0) - (_col0 * -563)) (type: bigint), _col4 (type: tinyint), _col4 (type: tinyint), (UDFToDouble(_col4) * (- (UDFToDouble(((_col0 * -563) % _col0)) / _col2))) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT SUM(cint),
(SUM(cint) * -563),
(-3728 + SUM(cint)),
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_5.q.out b/ql/src/test/results/clientpositive/vectorization_5.q.out
index 914a626..33707c7 100644
--- a/ql/src/test/results/clientpositive/vectorization_5.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_5.q.out
@@ -1,3 +1,147 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT MAX(csmallint),
+ (MAX(csmallint) * -75),
+ COUNT(*),
+ ((MAX(csmallint) * -75) / COUNT(*)),
+ (6981 * MAX(csmallint)),
+ MIN(csmallint),
+ (-(MIN(csmallint))),
+ (197 % ((MAX(csmallint) * -75) / COUNT(*))),
+ SUM(cint),
+ MAX(ctinyint),
+ (-(MAX(ctinyint))),
+ ((-(MAX(ctinyint))) + MAX(ctinyint))
+FROM alltypesorc
+WHERE (((cboolean2 IS NOT NULL)
+ AND (cstring1 LIKE '%b%'))
+ OR ((ctinyint = cdouble)
+ AND ((ctimestamp2 IS NOT NULL)
+ AND (cstring2 LIKE 'a'))))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT MAX(csmallint),
+ (MAX(csmallint) * -75),
+ COUNT(*),
+ ((MAX(csmallint) * -75) / COUNT(*)),
+ (6981 * MAX(csmallint)),
+ MIN(csmallint),
+ (-(MIN(csmallint))),
+ (197 % ((MAX(csmallint) * -75) / COUNT(*))),
+ SUM(cint),
+ MAX(ctinyint),
+ (-(MAX(ctinyint))),
+ ((-(MAX(ctinyint))) + MAX(ctinyint))
+FROM alltypesorc
+WHERE (((cboolean2 IS NOT NULL)
+ AND (cstring1 LIKE '%b%'))
+ OR ((ctinyint = cdouble)
+ AND ((ctimestamp2 IS NOT NULL)
+ AND (cstring2 LIKE 'a'))))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 11) -> boolean, FilterStringColLikeStringScalar(col 6, pattern %b%) -> boolean) -> boolean, FilterExprAndExpr(children: FilterDoubleColEqualDoubleColumn(col 12, col 5)(children: CastLongToDouble(col 0) -> 12:double) -> boolean, SelectColumnIsNotNull(col 9) -> boolean, FilterStringColLikeStringScalar(col 7, pattern a) -> boolean) -> boolean) -> boolean
+ predicate: ((cboolean2 is not null and (cstring1 like '%b%')) or ((UDFToDouble(ctinyint) = cdouble) and ctimestamp2 is not null and (cstring2 like 'a'))) (type: boolean)
+ Statistics: Num rows: 9216 Data size: 1981473 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int)
+ outputColumnNames: ctinyint, csmallint, cint
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 2]
+ Statistics: Num rows: 9216 Data size: 1981473 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(csmallint), count(), min(csmallint), sum(cint), max(ctinyint)
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1) -> smallint, VectorUDAFCountStar(*) -> bigint, VectorUDAFMinLong(col 1) -> smallint, VectorUDAFSumLong(col 2) -> bigint, VectorUDAFMaxLong(col 0) -> tinyint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3, 4]
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: tinyint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 5, 6, 7, 9, 11]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), max(VALUE._col4)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: smallint), (UDFToInteger(_col0) * -75) (type: int), _col1 (type: bigint), (UDFToDouble((UDFToInteger(_col0) * -75)) / UDFToDouble(_col1)) (type: double), (6981 * UDFToInteger(_col0)) (type: int), _col2 (type: smallint), (- _col2) (type: smallint), (197.0 % (UDFToDouble((UDFToInteger(_col0) * -75)) / UDFToDouble(_col1))) (type: double), _col3 (type: bigint), _col4 (type: tinyint), (- _col4) (type: tinyint), ((- _col4) + _col4) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT MAX(csmallint),
(MAX(csmallint) * -75),
COUNT(*),
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_6.q.out b/ql/src/test/results/clientpositive/vectorization_6.q.out
index 13897f6..9f3da46 100644
--- a/ql/src/test/results/clientpositive/vectorization_6.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_6.q.out
@@ -1,3 +1,111 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cboolean1,
+ cfloat,
+ cstring1,
+ (988888 * csmallint),
+ (-(csmallint)),
+ (-(cfloat)),
+ (-26.28 / cfloat),
+ (cfloat * 359),
+ (cint % ctinyint),
+ (-(cdouble)),
+ (ctinyint - -75),
+ (762 * (cint % ctinyint))
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND ((((cboolean1 <= 0)
+ AND (cboolean2 >= cboolean1))
+ OR ((cbigint IS NOT NULL)
+ AND ((cstring2 LIKE '%a')
+ OR (cfloat <= -257))))))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cboolean1,
+ cfloat,
+ cstring1,
+ (988888 * csmallint),
+ (-(csmallint)),
+ (-(cfloat)),
+ (-26.28 / cfloat),
+ (cfloat * 359),
+ (cint % ctinyint),
+ (-(cdouble)),
+ (ctinyint - -75),
+ (762 * (cint % ctinyint))
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND ((((cboolean1 <= 0)
+ AND (cboolean2 >= cboolean1))
+ OR ((cbigint IS NOT NULL)
+ AND ((cstring2 LIKE '%a')
+ OR (cfloat <= -257))))))
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0, val 0) -> boolean, FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10, val 0) -> boolean, FilterLongColGreaterEqualLongColumn(col 11, col 10) -> boolean) -> boolean, FilterExprAndExpr(children: SelectColumnIsNotNull(col 3) -> boolean, FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7, pattern %a) -> boolean, FilterDoubleColLessEqualDoubleScalar(col 4, val -257.0) -> boolean) -> boolean) -> boolean) -> boolean) -> boolean
+ predicate: ((ctinyint <> 0) and (((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (cbigint is not null and ((cstring2 like '%a') or (cfloat <= -257))))) (type: boolean)
+ Statistics: Num rows: 11605 Data size: 2495116 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28 / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [10, 4, 6, 12, 13, 14, 15, 16, 17, 18, 19, 21]
+ selectExpressions: LongScalarMultiplyLongColumn(val 988888, col 1)(children: col 1) -> 12:long, LongColUnaryMinus(col 1) -> 13:long, DoubleColUnaryMinus(col 4) -> 14:double, DoubleScalarDivideDoubleColumn(val -26.28, col 4)(children: col 4) -> 15:double, DoubleColMultiplyDoubleScalar(col 4, val 359.0) -> 16:double, LongColModuloLongColumn(col 2, col 0)(children: col 0) -> 17:long, DoubleColUnaryMinus(col 5) -> 18:double, LongColSubtractLongScalar(col 0, val -75)(children: col 0) -> 19:long, LongScalarMultiplyLongColumn(val 762, col 20)(children: LongColModuloLongColumn(col 2, col 0)(children: col 0) -> 20:long) -> 21:long
+ Statistics: Num rows: 11605 Data size: 2495116 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 11605 Data size: 2495116 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 10, 11]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint, bigint, double, double, double, bigint, double, bigint, bigint, bigint
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: SELECT cboolean1,
cfloat,
cstring1,
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_7.q.out b/ql/src/test/results/clientpositive/vectorization_7.q.out
index c05fee0..218d307 100644
--- a/ql/src/test/results/clientpositive/vectorization_7.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_7.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cboolean1,
cbigint,
csmallint,
@@ -25,7 +25,7 @@ WHERE ((ctinyint != 0)
ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
LIMIT 25
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cboolean1,
cbigint,
csmallint,
@@ -105,6 +105,12 @@ STAGE PLANS:
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 5, 6, 7, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_8.q.out b/ql/src/test/results/clientpositive/vectorization_8.q.out
index ce2a4b5..e56fb53 100644
--- a/ql/src/test/results/clientpositive/vectorization_8.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_8.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT ctimestamp1,
cdouble,
cboolean1,
@@ -23,7 +23,7 @@ WHERE (((cstring2 IS NOT NULL)
ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
LIMIT 20
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT ctimestamp1,
cdouble,
cboolean1,
@@ -101,6 +101,12 @@ STAGE PLANS:
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [2, 3, 4, 5, 6, 7, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: double, double, double, double, double, double, double, double, double, double, double
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_9.q.out b/ql/src/test/results/clientpositive/vectorization_9.q.out
index 2e3a34d..930b476 100644
--- a/ql/src/test/results/clientpositive/vectorization_9.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_9.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: EXPLAIN VECTORIZATION
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cstring1,
cdouble,
ctimestamp1,
@@ -18,7 +18,7 @@ WHERE ((cstring2 LIKE '%b%')
OR (cstring1 < 'a')))
GROUP BY cstring1, cdouble, ctimestamp1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN VECTORIZATION
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT cstring1,
cdouble,
ctimestamp1,
@@ -53,15 +53,35 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 7, pattern %b%) -> boolean, FilterExprOrExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5, val -1.389) -> boolean, FilterStringGroupColLessStringScalar(col 6, val a) -> boolean) -> boolean) -> boolean
predicate: ((cstring2 like '%b%') and ((cdouble >= -1.389) or (cstring1 < 'a'))) (type: boolean)
Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), cstring1 (type: string), ctimestamp1 (type: timestamp)
outputColumnNames: cdouble, cstring1, ctimestamp1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [5, 6, 8]
Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(cdouble), stddev_samp(cdouble), min(cdouble)
+ Group By Vectorization:
+ aggregators: VectorUDAFCount(col 5) -> bigint, VectorUDAFStdSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMinDouble(col 5) -> double
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 5, col 6, col 8
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2]
keys: cdouble (type: double), cstring1 (type: string), ctimestamp1 (type: timestamp)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -70,17 +90,27 @@ STAGE PLANS:
key expressions: _col0 (type: double), _col1 (type: string), _col2 (type: timestamp)
sort order: +++
Map-reduce partition columns: _col0 (type: double), _col1 (type: string), _col2 (type: timestamp)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: double)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [5, 6, 7, 8]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
@@ -88,6 +118,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0), stddev_samp(VALUE._col1), min(VALUE._col2)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
keys: KEY._col0 (type: double), KEY._col1 (type: string), KEY._col2 (type: timestamp)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_limit.q.out b/ql/src/test/results/clientpositive/vectorization_limit.q.out
index 7381294..b46e6ef 100644
--- a/ql/src/test/results/clientpositive/vectorization_limit.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_limit.q.out
@@ -221,13 +221,14 @@ STAGE PLANS:
Group By Operator
aggregations: avg(_col1)
Group By Vectorization:
- aggregators: VectorUDAFAvgDouble(col 12) -> struct<count:bigint,sum:double>
+ aggregators: VectorUDAFAvgDouble(col 12) -> struct<count:bigint,sum:double,input:double>
className: VectorGroupByOperator
- vectorOutput: false
+ groupByMode: HASH
+ vectorOutput: true
keyExpressions: col 0
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0]
- vectorOutputConditionsNotMet: Vector output of VectorUDAFAvgDouble(col 12) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false
keys: _col0 (type: tinyint)
mode: hash
outputColumnNames: _col0, _col1
@@ -236,6 +237,11 @@ STAGE PLANS:
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: struct<count:bigint,sum:double,input:double>)
@@ -243,7 +249,7 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -262,8 +268,10 @@ STAGE PLANS:
Group By Operator
aggregations: avg(VALUE._col0)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
keys: KEY._col0 (type: tinyint)
mode: mergepartial
@@ -349,9 +357,11 @@ STAGE PLANS:
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
keyExpressions: col 0
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: []
keys: ctinyint (type: tinyint)
mode: hash
@@ -389,8 +399,10 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
keys: KEY._col0 (type: tinyint)
mode: mergepartial
@@ -478,9 +490,11 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFCount(col 5) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
keyExpressions: col 0, col 5
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0]
keys: ctinyint (type: tinyint), cdouble (type: double)
mode: hash
@@ -519,8 +533,10 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
keys: KEY._col0 (type: tinyint)
mode: mergepartial
@@ -636,9 +652,11 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFSumLong(col 0) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
keyExpressions: col 5
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0]
keys: cdouble (type: double)
mode: hash
@@ -677,8 +695,10 @@ STAGE PLANS:
Group By Operator
aggregations: sum(VALUE._col0)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
keys: KEY._col0 (type: double)
mode: mergepartial
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
index 664a9ba..183cbdc 100644
--- a/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
@@ -39,7 +39,7 @@ STAGE PLANS:
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_case.q.out b/ql/src/test/results/clientpositive/vectorized_case.q.out
index b1b5e54..0a2e803 100644
--- a/ql/src/test/results/clientpositive/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_case.q.out
@@ -276,8 +276,10 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1]
mode: hash
outputColumnNames: _col0, _col1
@@ -308,8 +310,10 @@ STAGE PLANS:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1
@@ -387,8 +391,10 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1]
mode: hash
outputColumnNames: _col0, _col1
@@ -419,8 +425,10 @@ STAGE PLANS:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out b/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
index 4248d08..b7ac3f9 100644
--- a/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_date_funcs.q.out
@@ -1239,8 +1239,10 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFMinLong(col 0) -> date, VectorUDAFMaxLong(col 0) -> date, VectorUDAFCount(col 0) -> bigint, VectorUDAFCountStar(*) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1, 2, 3]
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
@@ -1271,8 +1273,10 @@ STAGE PLANS:
Group By Operator
aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
index 81292ec..1fe1c69 100644
--- a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
@@ -16,9 +16,11 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dtest
POSTHOOK: Lineage: dtest.a SCRIPT []
POSTHOOK: Lineage: dtest.b SIMPLE []
-PREHOOK: query: explain vectorization select sum(distinct a), count(distinct a) from dtest
+PREHOOK: query: explain vectorization detail
+select sum(distinct a), count(distinct a) from dtest
PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization select sum(distinct a), count(distinct a) from dtest
+POSTHOOK: query: explain vectorization detail
+select sum(distinct a), count(distinct a) from dtest
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -35,13 +37,29 @@ STAGE PLANS:
TableScan
alias: dtest
Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
Select Operator
expressions: a (type: int)
outputColumnNames: a
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0]
Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(DISTINCT a), count(DISTINCT a)
bucketGroup: true
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0) -> bigint, VectorUDAFCount(col 0) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 0
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1]
keys: a (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
@@ -49,6 +67,11 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
@@ -59,6 +82,11 @@ STAGE PLANS:
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0]
+ dataColumns: a:int, b:int
+ partitionColumnCount: 0
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
@@ -66,6 +94,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: sum(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -92,9 +126,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@dtest
#### A masked pattern was here ####
300 1
-PREHOOK: query: explain vectorization select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc
+PREHOOK: query: explain vectorization detail
+select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc
PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc
+POSTHOOK: query: explain vectorization detail
+select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -111,12 +147,28 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Select Operator
expressions: cint (type: int)
outputColumnNames: cint
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2]
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(DISTINCT cint), count(DISTINCT cint), avg(DISTINCT cint), std(DISTINCT cint)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 2) -> bigint, VectorUDAFCount(col 2) -> bigint, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ vectorOutput: true
+ keyExpressions: col 2
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumns: [0, 1, 2, 3]
keys: cint (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
@@ -124,16 +176,26 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [2]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
@@ -141,6 +203,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: sum(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), avg(DISTINCT KEY._col0:2._col0), std(DISTINCT KEY._col0:3._col0)
+ Group By Vectorization:
+ groupByMode: MERGEPARTIAL
+ vectorOutput: false
+ native: false
+ vectorProcessingMode: NONE
+ projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out b/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
index 32210ad..b915e87 100644
--- a/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
@@ -88,24 +88,30 @@ STAGE PLANS:
Group By Operator
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 0) -> bigint, VectorUDAFMaxLong(col 1) -> int, VectorUDAFMinLong(col 0) -> int, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double>
+ aggregators: VectorUDAFCount(col 0) -> bigint, VectorUDAFMaxLong(col 1) -> int, VectorUDAFMinLong(col 0) -> int, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint>
className: VectorGroupByOperator
- vectorOutput: false
+ groupByMode: HASH
+ vectorOutput: true
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1, 2, 3]
- vectorOutputConditionsNotMet: Vector output of VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -120,8 +126,10 @@ STAGE PLANS:
Group By Operator
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out b/ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out
index 52aa05b..5334c16 100644
--- a/ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_mapjoin2.q.out
@@ -108,8 +108,10 @@ STAGE PLANS:
Group By Vectorization:
aggregators: VectorUDAFCount(ConstantVectorExpression(val 1) -> 0:long) -> bigint
className: VectorGroupByOperator
+ groupByMode: HASH
vectorOutput: true
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0]
mode: hash
outputColumnNames: _col0
@@ -142,8 +144,10 @@ STAGE PLANS:
Group By Operator
aggregations: count(VALUE._col0)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
index 46e51f7..e096c72 100644
--- a/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
@@ -351,13 +351,14 @@ STAGE PLANS:
Group By Operator
aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble), max(cdecimal)
Group By Vectorization:
- aggregators: VectorUDAFMaxLong(col 0) -> int, VectorUDAFMinLong(col 2) -> smallint, VectorUDAFCount(col 5) -> bigint, VectorUDAFAvgDouble(col 3) -> struct<count:bigint,sum:double>, VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDecimal(col 10) -> decimal(4,2)
+ aggregators: VectorUDAFMaxLong(col 0) -> int, VectorUDAFMinLong(col 2) -> smallint, VectorUDAFCount(col 5) -> bigint, VectorUDAFAvgDouble(col 3) -> struct<count:bigint,sum:double,input:double>, VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDecimal(col 10) -> decimal(4,2)
className: VectorGroupByOperator
- vectorOutput: false
+ groupByMode: HASH
+ vectorOutput: true
keyExpressions: col 1
native: false
+ vectorProcessingMode: HASH
projectedOutputColumns: [0, 1, 2, 3, 4, 5]
- vectorOutputConditionsNotMet: Vector output of VectorUDAFAvgDouble(col 3) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false
keys: ctinyint (type: tinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
@@ -366,13 +367,18 @@ STAGE PLANS:
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 22 Data size: 242 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: decimal(4,2))
Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- groupByVectorOutput: false
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
allNative: false
usesVectorUDFAdaptor: false
@@ -385,8 +391,10 @@ STAGE PLANS:
Group By Operator
aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4), max(VALUE._col5)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
keys: KEY._col0 (type: tinyint)
mode: mergepartial
http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
index d42369f..d1d5e55 100644
--- a/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
@@ -73,8 +73,10 @@ STAGE PLANS:
Group By Operator
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
Group By Vectorization:
+ groupByMode: HASH
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
@@ -90,16 +92,27 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3]
Reduce Output Operator
sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
+ Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- notVectorizedReason: Value expression for REDUCESINK operator: Data type struct<count:bigint,sum:double,input:int> of Column[_col3] not supported
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Vectorization:
enabled: false
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
@@ -108,8 +121,10 @@ STAGE PLANS:
Group By Operator
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
Group By Vectorization:
+ groupByMode: MERGEPARTIAL
vectorOutput: false
native: false
+ vectorProcessingMode: NONE
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3