You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/03/22 16:57:32 UTC
[09/34] hive git commit: HIVE-18979: Enable
AggregateReduceFunctionsRule from Calcite (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/5cb8867b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
index 98c709c..d37a27e 100644
--- a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
@@ -249,26 +249,27 @@ STAGE PLANS:
native: true
vectorizationSchemaColumns: [0:cdouble:double, 1:cdecimal1:decimal(20,10), 2:cdecimal2:decimal(23,14), 3:cint:int, 4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
- expressions: cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)), cint (type: int)
- outputColumnNames: cdecimal1, cdecimal2, cint
+ expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)), UDFToDouble(cdecimal1) (type: double), (UDFToDouble(cdecimal1) * UDFToDouble(cdecimal1)) (type: double), UDFToDouble(cdecimal2) (type: double), (UDFToDouble(cdecimal2) * UDFToDouble(cdecimal2)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [1, 2, 3]
+ projectedOutputColumnNums: [3, 1, 2, 5, 8, 6, 10]
+ selectExpressions: CastDecimalToDouble(col 1:decimal(20,10)) -> 5:double, DoubleColMultiplyDoubleColumn(col 6:double, col 7:double)(children: CastDecimalToDouble(col 1:decimal(20,10)) -> 6:double, CastDecimalToDouble(col 1:decimal(20,10)) -> 7:double) -> 8:double, CastDecimalToDouble(col 2:decimal(23,14)) -> 6:double, DoubleColMultiplyDoubleColumn(col 7:double, col 9:double)(children: CastDecimalToDouble(col 2:decimal(23,14)) -> 7:double, CastDecimalToDouble(col 2:decimal(23,14)) -> 9:double) -> 10:double
Statistics: Num rows: 12289 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), avg(cdecimal1), stddev_pop(cdecimal1), stddev_samp(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), avg(cdecimal2), stddev_pop(cdecimal2), stddev_samp(cdecimal2), count()
+ aggregations: count(_col1), max(_col1), min(_col1), sum(_col1), sum(_col4), sum(_col3), count(_col2), max(_col2), min(_col2), sum(_col2), sum(_col6), sum(_col5), count()
Group By Vectorization:
- aggregators: VectorUDAFCount(col 1:decimal(20,10)) -> bigint, VectorUDAFMaxDecimal(col 1:decimal(20,10)) -> decimal(20,10), VectorUDAFMinDecimal(col 1:decimal(20,10)) -> decimal(20,10), VectorUDAFSumDecimal(col 1:decimal(20,10)) -> decimal(30,10), VectorUDAFAvgDecimal(col 1:decimal(20,10)) -> struct<count:bigint,sum:decimal(30,10),input:decimal(20,10)>, VectorUDAFVarDecimal(col 1:decimal(20,10)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_pop, VectorUDAFVarDecimal(col 1:decimal(20,10)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_samp, VectorUDAFCount(col 2:decimal(23,14)) -> bigint, VectorUDAFMaxDecimal(col 2:decimal(23,14)) -> decimal(23,14), VectorUDAFMinDecimal(col 2:decimal(23,14)) -> decimal(23,14), VectorUDAFSumDecimal(col 2:decimal(23,14)) -> decimal(33,14), VectorUDAFAvgDecimal(col 2:decimal(23,14)) -> struct<count:bigint,sum:decimal(33,14),input:decimal(23,14)>, VectorUDAFVarDecimal(col 2:decima
l(23,14)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_pop, VectorUDAFVarDecimal(col 2:decimal(23,14)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_samp, VectorUDAFCountStar(*) -> bigint
+ aggregators: VectorUDAFCount(col 1:decimal(20,10)) -> bigint, VectorUDAFMaxDecimal(col 1:decimal(20,10)) -> decimal(20,10), VectorUDAFMinDecimal(col 1:decimal(20,10)) -> decimal(20,10), VectorUDAFSumDecimal(col 1:decimal(20,10)) -> decimal(30,10), VectorUDAFSumDouble(col 8:double) -> double, VectorUDAFSumDouble(col 5:double) -> double, VectorUDAFCount(col 2:decimal(23,14)) -> bigint, VectorUDAFMaxDecimal(col 2:decimal(23,14)) -> decimal(23,14), VectorUDAFMinDecimal(col 2:decimal(23,14)) -> decimal(23,14), VectorUDAFSumDecimal(col 2:decimal(23,14)) -> decimal(33,14), VectorUDAFSumDouble(col 10:double) -> double, VectorUDAFSumDouble(col 6:double) -> double, VectorUDAFCountStar(*) -> bigint
className: VectorGroupByOperator
groupByMode: HASH
keyExpressions: col 3:int
native: false
vectorProcessingMode: HASH
- projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
- keys: cint (type: int)
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+ keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 12289 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
@@ -279,9 +280,9 @@ STAGE PLANS:
keyColumnNums: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+ valueColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
Statistics: Num rows: 12289 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: struct<count:bigint,sum:decimal(30,10),input:decimal(20,10)>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: struct<count:bigint,sum:decimal(33,14),input:decimal(23,14)>), _col13 (type: struct<count:bigint,sum:double,variance:double>), _col14 (type: struct<count:bigint,sum:double,variance:double>), _col15 (type: bigint)
+ value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: decimal(23,14)), _col9 (type: decimal(23,14)), _col10 (type: decimal(33,14)), _col11 (type: double), _col12 (type: double), _col13 (type: bigint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -297,29 +298,58 @@ STAGE PLANS:
includeColumns: [1, 2, 3]
dataColumns: cdouble:double, cdecimal1:decimal(20,10), cdecimal2:decimal(23,14), cint:int
partitionColumnCount: 0
- scratchColumnTypeNames: []
+ scratchColumnTypeNames: [double, double, double, double, double, double]
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 14
+ dataColumns: KEY._col0:int, VALUE._col0:bigint, VALUE._col1:decimal(20,10), VALUE._col2:decimal(20,10), VALUE._col3:decimal(30,10), VALUE._col4:double, VALUE._col5:double, VALUE._col6:bigint, VALUE._col7:decimal(23,14), VALUE._col8:decimal(23,14), VALUE._col9:decimal(33,14), VALUE._col10:double, VALUE._col11:double, VALUE._col12:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), avg(VALUE._col4), stddev_pop(VALUE._col5), stddev_samp(VALUE._col6), count(VALUE._col7), max(VALUE._col8), min(VALUE._col9), sum(VALUE._col10), avg(VALUE._col11), stddev_pop(VALUE._col12), stddev_samp(VALUE._col13), count(VALUE._col14)
+ aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), count(VALUE._col6), max(VALUE._col7), min(VALUE._col8), sum(VALUE._col9), sum(VALUE._col10), sum(VALUE._col11), count(VALUE._col12)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint, VectorUDAFMaxDecimal(col 2:decimal(20,10)) -> decimal(20,10), VectorUDAFMinDecimal(col 3:decimal(20,10)) -> decimal(20,10), VectorUDAFSumDecimal(col 4:decimal(30,10)) -> decimal(30,10), VectorUDAFSumDouble(col 5:double) -> double, VectorUDAFSumDouble(col 6:double) -> double, VectorUDAFCountMerge(col 7:bigint) -> bigint, VectorUDAFMaxDecimal(col 8:decimal(23,14)) -> decimal(23,14), VectorUDAFMinDecimal(col 9:decimal(23,14)) -> decimal(23,14), VectorUDAFSumDecimal(col 10:decimal(33,14)) -> decimal(33,14), VectorUDAFSumDouble(col 11:double) -> double, VectorUDAFSumDouble(col 12:double) -> double, VectorUDAFCountMerge(col 13:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
keys: KEY._col0 (type: int)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 6144 Data size: 1082441 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col15 > 1L) (type: boolean)
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterLongColGreaterLongScalar(col 13:bigint, val 1)
+ predicate: (_col13 > 1L) (type: boolean)
Statistics: Num rows: 2048 Data size: 360813 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: decimal(24,14)), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: decimal(27,18)), _col13 (type: double), _col14 (type: double)
+ expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), (CAST( _col4 AS decimal(24,14)) / _col1) (type: decimal(38,28)), power(((_col5 - ((_col6 * _col6) / _col1)) / _col1), 0.5) (type: double), power(((_col5 - ((_col6 * _col6) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END), 0.5) (type: double), _col7 (type: bigint), _col8 (type: decimal(23,14)), _col9 (type: decimal(23,14)), _col10 (type: decimal(33,14)), (CAST( _col10 AS decimal(27,18)) / _col7) (type: decimal(38,29)), power(((_col11 - ((_col12 * _col12) / _col7)) / _col7), 0.5) (type: double), power(((_col11 - ((_col12 * _col12) / _col7)) / CASE WHEN ((_col7 = 1L)) THEN (null) ELSE ((_col7 - 1)) END), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 16, 17, 18, 7, 8, 9, 10, 24, 19, 25]
+ selectExpressions: DecimalColDivideDecimalColumn(col 14:decimal(24,14), col 15:decimal(19,0))(children: CastDecimalToDecimal(col 4:decimal(30,10)) -> 14:decimal(24,14), CastLongToDecimal(col 1:bigint) -> 15:decimal(19,0)) -> 16:decimal(38,28), FuncPowerDoubleToDouble(col 18:double)(children: DoubleColDivideLongColumn(col 17:double, col 1:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 18:double)(children: DoubleColDivideLongColumn(col 17:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 17:double) -> 18:double) -> 17:double) -> 18:double) -> 17:double, FuncPowerDoubleToDouble(col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 22:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 18:double) -> 19:double) -> 18:double, IfEx
prNullCondExpr(col 20:boolean, null, col 21:bigint)(children: LongColEqualLongScalar(col 1:bigint, val 1) -> 20:boolean, LongColSubtractLongScalar(col 1:bigint, val 1) -> 21:bigint) -> 22:bigint) -> 19:double) -> 18:double, DecimalColDivideDecimalColumn(col 23:decimal(27,18), col 15:decimal(19,0))(children: CastDecimalToDecimal(col 10:decimal(33,14)) -> 23:decimal(27,18), CastLongToDecimal(col 7:bigint) -> 15:decimal(19,0)) -> 24:decimal(38,29), FuncPowerDoubleToDouble(col 25:double)(children: DoubleColDivideLongColumn(col 19:double, col 7:bigint)(children: DoubleColSubtractDoubleColumn(col 11:double, col 25:double)(children: DoubleColDivideLongColumn(col 19:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 12:double, col 12:double) -> 19:double) -> 25:double) -> 19:double) -> 25:double) -> 19:double, FuncPowerDoubleToDouble(col 26:double)(children: DoubleColDivideLongColumn(col 25:double, col 28:bigint)(children: DoubleColSubtractDoubleColumn(col 11:double, col 26:d
ouble)(children: DoubleColDivideLongColumn(col 25:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 12:double, col 12:double) -> 25:double) -> 26:double) -> 25:double, IfExprNullCondExpr(col 22:boolean, null, col 27:bigint)(children: LongColEqualLongScalar(col 7:bigint, val 1) -> 22:boolean, LongColSubtractLongScalar(col 7:bigint, val 1) -> 27:bigint) -> 28:bigint) -> 26:double) -> 25:double
Statistics: Num rows: 2048 Data size: 360813 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 2048 Data size: 360813 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -350,14 +380,14 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
--3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.21109769200000 -4033.44576923076900 6967702.86724384584710 1161283.811207307641183333 2604201.2704476737 2852759.5602156054
--563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 -1941.63641486490000 1426.0153418918999 2016.6902366556308 2 -617.56077692307690 -4033.44576923076900 -4651.00654615384590 -2325.503273076922950000 1707.9424961538462 2415.395441814127
-253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274
-528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984
-626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603
-6981 3 5831542.2692483780 -515.6210729730 5830511.0271024320 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.21109769200000 -617.56077692307690 6983219.08954384584620 2327739.696514615282066667 3292794.4113115156 4032833.0678006653
-762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2916536.74432689170000 2915005.5249214866 4122440.3477364695 2 6984454.21109769200000 1833.94569230769250 6986288.15678999969250 3493144.078394999846250000 3491310.1327026924 4937458.140118758
-NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 1956.576923076922966667 6821.495748565159 6822.606289190924
+-3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 969592.6735247296333333333333333333 2174330.209240386 2381859.406131774 6 6984454.21109769200000 -4033.44576923076900 6967702.86724384584710 1161283.81120730764118333333333333333 2604201.2704476737 2852759.5602156054
+-563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 -1941.6364148649000000000000000000 1426.0153418918997 2016.6902366556305 2 -617.56077692307690 -4033.44576923076900 -4651.00654615384590 -2325.50327307692295000000000000000 1707.9424961538462 2415.395441814127
+253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.3399236697630859375000000000 5708.956347886203 5711.745967572781 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 -406.42835967548079188476562500000 6837.632716002931 6840.973851172272
+528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 11373.4109968243230468750000000000 257528.9298820665 257654.7686043977 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 13621.96562480769168948242187500000 308443.1074570797 308593.82484083937
+626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.2939966110631835937500000000 5742.091453237337 5744.897264034264 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 12.32914804687497798828125000000 6877.318722794881 6880.679250101608
+6981 3 5831542.2692483780 -515.6210729730 5830511.0271024320 1943503.6757008106666666666666666667 2749258.455012492 3367140.1929065133 3 6984454.21109769200000 -617.56077692307690 6983219.08954384584620 2327739.69651461528206666666666666667 3292794.4113115156 4032833.0678006653
+762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2916536.7443268917000000000000000000 2915005.5249214866 4122440.3477364695 2 6984454.21109769200000 1833.94569230769250 6986288.15678999969250 3493144.07839499984625000000000000000 3491310.1327026924 4937458.140118757
+NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.6081081080666666666666666667 5695.483082135323 5696.410307714464 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 1956.57692307692296666666666666667 6821.495748565151 6822.606289190915
PREHOOK: query: CREATE TABLE decimal_vgby_small STORED AS TEXTFILE AS
SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(11,5)) AS cdecimal1,
CAST (((cdouble*9.3)/13) AS DECIMAL(16,0)) AS cdecimal2,
@@ -628,26 +658,27 @@ STAGE PLANS:
native: true
vectorizationSchemaColumns: [0:cdouble:double, 1:cdecimal1:decimal(11,5)/DECIMAL_64, 2:cdecimal2:decimal(16,0)/DECIMAL_64, 3:cint:int, 4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
- expressions: cdecimal1 (type: decimal(11,5)), cdecimal2 (type: decimal(16,0)), cint (type: int)
- outputColumnNames: cdecimal1, cdecimal2, cint
+ expressions: cint (type: int), cdecimal1 (type: decimal(11,5)), cdecimal2 (type: decimal(16,0)), UDFToDouble(cdecimal1) (type: double), (UDFToDouble(cdecimal1) * UDFToDouble(cdecimal1)) (type: double), UDFToDouble(cdecimal2) (type: double), (UDFToDouble(cdecimal2) * UDFToDouble(cdecimal2)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [1, 2, 3]
+ projectedOutputColumnNums: [3, 1, 2, 6, 9, 7, 12]
+ selectExpressions: CastDecimalToDouble(col 5:decimal(11,5))(children: ConvertDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> 5:decimal(11,5)) -> 6:double, DoubleColMultiplyDoubleColumn(col 7:double, col 8:double)(children: CastDecimalToDouble(col 5:decimal(11,5))(children: ConvertDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> 5:decimal(11,5)) -> 7:double, CastDecimalToDouble(col 5:decimal(11,5))(children: ConvertDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> 5:decimal(11,5)) -> 8:double) -> 9:double, CastDecimalToDouble(col 10:decimal(16,0))(children: ConvertDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> 10:decimal(16,0)) -> 7:double, DoubleColMultiplyDoubleColumn(col 8:double, col 11:double)(children: CastDecimalToDouble(col 10:decimal(16,0))(children: ConvertDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> 10:decimal(16,0)) -> 8:double, CastDecimalToDouble(col 10:decimal(16,0))(children: ConvertDecimal64ToDecimal(col 2:decima
l(16,0)/DECIMAL_64) -> 10:decimal(16,0)) -> 11:double) -> 12:double
Statistics: Num rows: 12289 Data size: 346472 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), avg(cdecimal1), stddev_pop(cdecimal1), stddev_samp(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), avg(cdecimal2), stddev_pop(cdecimal2), stddev_samp(cdecimal2), count()
+ aggregations: count(_col1), max(_col1), min(_col1), sum(_col1), sum(_col4), sum(_col3), count(_col2), max(_col2), min(_col2), sum(_col2), sum(_col6), sum(_col5), count()
Group By Vectorization:
- aggregators: VectorUDAFCount(col 1:decimal(11,5)/DECIMAL_64) -> bigint, VectorUDAFMaxDecimal64(col 1:decimal(11,5)/DECIMAL_64) -> decimal(11,5)/DECIMAL_64, VectorUDAFMinDecimal64(col 1:decimal(11,5)/DECIMAL_64) -> decimal(11,5)/DECIMAL_64, VectorUDAFSumDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> decimal(21,5), VectorUDAFAvgDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> struct<count:bigint,sum:decimal(21,5),input:decimal(11,5)>, VectorUDAFVarDecimal(ConvertDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> 5:decimal(11,5)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_pop, VectorUDAFVarDecimal(ConvertDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> 6:decimal(11,5)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_samp, VectorUDAFCount(col 2:decimal(16,0)/DECIMAL_64) -> bigint, VectorUDAFMaxDecimal64(col 2:decimal(16,0)/DECIMAL_64) -> decimal(16,0)/DECIMAL_64, VectorUDAFMinDecimal64(col 2
:decimal(16,0)/DECIMAL_64) -> decimal(16,0)/DECIMAL_64, VectorUDAFSumDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> decimal(26,0), VectorUDAFAvgDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> struct<count:bigint,sum:decimal(26,0),input:decimal(16,0)>, VectorUDAFVarDecimal(ConvertDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> 7:decimal(16,0)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_pop, VectorUDAFVarDecimal(ConvertDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> 8:decimal(16,0)) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_samp, VectorUDAFCountStar(*) -> bigint
+ aggregators: VectorUDAFCount(col 1:decimal(11,5)/DECIMAL_64) -> bigint, VectorUDAFMaxDecimal64(col 1:decimal(11,5)/DECIMAL_64) -> decimal(11,5)/DECIMAL_64, VectorUDAFMinDecimal64(col 1:decimal(11,5)/DECIMAL_64) -> decimal(11,5)/DECIMAL_64, VectorUDAFSumDecimal64ToDecimal(col 1:decimal(11,5)/DECIMAL_64) -> decimal(21,5), VectorUDAFSumDouble(col 9:double) -> double, VectorUDAFSumDouble(col 6:double) -> double, VectorUDAFCount(col 2:decimal(16,0)/DECIMAL_64) -> bigint, VectorUDAFMaxDecimal64(col 2:decimal(16,0)/DECIMAL_64) -> decimal(16,0)/DECIMAL_64, VectorUDAFMinDecimal64(col 2:decimal(16,0)/DECIMAL_64) -> decimal(16,0)/DECIMAL_64, VectorUDAFSumDecimal64ToDecimal(col 2:decimal(16,0)/DECIMAL_64) -> decimal(26,0), VectorUDAFSumDouble(col 12:double) -> double, VectorUDAFSumDouble(col 7:double) -> double, VectorUDAFCountStar(*) -> bigint
className: VectorGroupByOperator
groupByMode: HASH
keyExpressions: col 3:int
native: false
vectorProcessingMode: HASH
- projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
- keys: cint (type: int)
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+ keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 12289 Data size: 346472 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
@@ -658,9 +689,9 @@ STAGE PLANS:
keyColumnNums: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+ valueColumnNums: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
Statistics: Num rows: 12289 Data size: 346472 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint), _col2 (type: decimal(11,5)), _col3 (type: decimal(11,5)), _col4 (type: decimal(21,5)), _col5 (type: struct<count:bigint,sum:decimal(21,5),input:decimal(11,5)>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: bigint), _col9 (type: decimal(16,0)), _col10 (type: decimal(16,0)), _col11 (type: decimal(26,0)), _col12 (type: struct<count:bigint,sum:decimal(26,0),input:decimal(16,0)>), _col13 (type: struct<count:bigint,sum:double,variance:double>), _col14 (type: struct<count:bigint,sum:double,variance:double>), _col15 (type: bigint)
+ value expressions: _col1 (type: bigint), _col2 (type: decimal(11,5)), _col3 (type: decimal(11,5)), _col4 (type: decimal(21,5)), _col5 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: decimal(16,0)), _col9 (type: decimal(16,0)), _col10 (type: decimal(26,0)), _col11 (type: double), _col12 (type: double), _col13 (type: bigint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -676,29 +707,58 @@ STAGE PLANS:
includeColumns: [1, 2, 3]
dataColumns: cdouble:double, cdecimal1:decimal(11,5)/DECIMAL_64, cdecimal2:decimal(16,0)/DECIMAL_64, cint:int
partitionColumnCount: 0
- scratchColumnTypeNames: [decimal(11,5), decimal(11,5), decimal(16,0), decimal(16,0)]
+ scratchColumnTypeNames: [decimal(11,5), double, double, double, double, decimal(16,0), double, double]
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ reduceColumnNullOrder: a
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 14
+ dataColumns: KEY._col0:int, VALUE._col0:bigint, VALUE._col1:decimal(11,5), VALUE._col2:decimal(11,5), VALUE._col3:decimal(21,5), VALUE._col4:double, VALUE._col5:double, VALUE._col6:bigint, VALUE._col7:decimal(16,0), VALUE._col8:decimal(16,0), VALUE._col9:decimal(26,0), VALUE._col10:double, VALUE._col11:double, VALUE._col12:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), avg(VALUE._col4), stddev_pop(VALUE._col5), stddev_samp(VALUE._col6), count(VALUE._col7), max(VALUE._col8), min(VALUE._col9), sum(VALUE._col10), avg(VALUE._col11), stddev_pop(VALUE._col12), stddev_samp(VALUE._col13), count(VALUE._col14)
+ aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), count(VALUE._col6), max(VALUE._col7), min(VALUE._col8), sum(VALUE._col9), sum(VALUE._col10), sum(VALUE._col11), count(VALUE._col12)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint, VectorUDAFMaxDecimal(col 2:decimal(11,5)) -> decimal(11,5), VectorUDAFMinDecimal(col 3:decimal(11,5)) -> decimal(11,5), VectorUDAFSumDecimal(col 4:decimal(21,5)) -> decimal(21,5), VectorUDAFSumDouble(col 5:double) -> double, VectorUDAFSumDouble(col 6:double) -> double, VectorUDAFCountMerge(col 7:bigint) -> bigint, VectorUDAFMaxDecimal(col 8:decimal(16,0)) -> decimal(16,0), VectorUDAFMinDecimal(col 9:decimal(16,0)) -> decimal(16,0), VectorUDAFSumDecimal(col 10:decimal(26,0)) -> decimal(26,0), VectorUDAFSumDouble(col 11:double) -> double, VectorUDAFSumDouble(col 12:double) -> double, VectorUDAFCountMerge(col 13:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
keys: KEY._col0 (type: int)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 6144 Data size: 173221 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col15 > 1L) (type: boolean)
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterLongColGreaterLongScalar(col 13:bigint, val 1)
+ predicate: (_col13 > 1L) (type: boolean)
Statistics: Num rows: 2048 Data size: 57740 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(11,5)), _col3 (type: decimal(11,5)), _col4 (type: decimal(21,5)), _col5 (type: decimal(15,9)), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: decimal(16,0)), _col10 (type: decimal(16,0)), _col11 (type: decimal(26,0)), _col12 (type: decimal(20,4)), _col13 (type: double), _col14 (type: double)
+ expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: decimal(11,5)), _col3 (type: decimal(11,5)), _col4 (type: decimal(21,5)), (CAST( _col4 AS decimal(15,9)) / _col1) (type: decimal(35,29)), power(((_col5 - ((_col6 * _col6) / _col1)) / _col1), 0.5) (type: double), power(((_col5 - ((_col6 * _col6) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END), 0.5) (type: double), _col7 (type: bigint), _col8 (type: decimal(16,0)), _col9 (type: decimal(16,0)), _col10 (type: decimal(26,0)), CAST( (CAST( _col10 AS decimal(20,4)) / _col7) AS decimal(20,4)) (type: decimal(20,4)), power(((_col11 - ((_col12 * _col12) / _col7)) / _col7), 0.5) (type: double), power(((_col11 - ((_col12 * _col12) / _col7)) / CASE WHEN ((_col7 = 1L)) THEN (null) ELSE ((_col7 - 1)) END), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 16, 17, 18, 7, 8, 9, 10, 23, 19, 25]
+ selectExpressions: DecimalColDivideDecimalColumn(col 14:decimal(15,9), col 15:decimal(19,0))(children: CastDecimalToDecimal(col 4:decimal(21,5)) -> 14:decimal(15,9), CastLongToDecimal(col 1:bigint) -> 15:decimal(19,0)) -> 16:decimal(35,29), FuncPowerDoubleToDouble(col 18:double)(children: DoubleColDivideLongColumn(col 17:double, col 1:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 18:double)(children: DoubleColDivideLongColumn(col 17:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 17:double) -> 18:double) -> 17:double) -> 18:double) -> 17:double, FuncPowerDoubleToDouble(col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 22:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 18:double) -> 19:double) -> 18:double, IfExprN
ullCondExpr(col 20:boolean, null, col 21:bigint)(children: LongColEqualLongScalar(col 1:bigint, val 1) -> 20:boolean, LongColSubtractLongScalar(col 1:bigint, val 1) -> 21:bigint) -> 22:bigint) -> 19:double) -> 18:double, CastDecimalToDecimal(col 24:decimal(38,22))(children: DecimalColDivideDecimalColumn(col 23:decimal(20,4), col 15:decimal(19,0))(children: CastDecimalToDecimal(col 10:decimal(26,0)) -> 23:decimal(20,4), CastLongToDecimal(col 7:bigint) -> 15:decimal(19,0)) -> 24:decimal(38,22)) -> 23:decimal(20,4), FuncPowerDoubleToDouble(col 25:double)(children: DoubleColDivideLongColumn(col 19:double, col 7:bigint)(children: DoubleColSubtractDoubleColumn(col 11:double, col 25:double)(children: DoubleColDivideLongColumn(col 19:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 12:double, col 12:double) -> 19:double) -> 25:double) -> 19:double) -> 25:double) -> 19:double, FuncPowerDoubleToDouble(col 26:double)(children: DoubleColDivideLongColumn(col 25:double, col 28:bi
gint)(children: DoubleColSubtractDoubleColumn(col 11:double, col 26:double)(children: DoubleColDivideLongColumn(col 25:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 12:double, col 12:double) -> 25:double) -> 26:double) -> 25:double, IfExprNullCondExpr(col 22:boolean, null, col 27:bigint)(children: LongColEqualLongScalar(col 7:bigint, val 1) -> 22:boolean, LongColSubtractLongScalar(col 7:bigint, val 1) -> 27:bigint) -> 28:bigint) -> 26:double) -> 25:double
Statistics: Num rows: 2048 Data size: 57740 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 2048 Data size: 57740 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -729,14 +789,14 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby_small
#### A masked pattern was here ####
--3728 5 -515.62107 -3367.65176 -13986.22811 -2797.245622000 1140.812276 1275.466899351126 6 6984454 -4033 6967704 1161284.0000 2604201.0914565204 2852759.364140621
--563 2 -515.62107 -3367.65176 -3883.27283 -1941.636415000 1426.0153450000003 2016.6902410511484 2 -618 -4033 -4651 -2325.5000 1707.5 2414.7696577520596
-253665376 1024 9767.00541 -9779.54865 -347484.08192 -339.339923750 5708.956347957812 5711.745967644425 1024 11698 -11713 -416183 -406.4287 6837.6426468206855 6840.983786842613
-528534767 1022 9777.75676 -9777.15946 -16711.67771 -16.351935137 5555.7621107931345 5558.482190324908 1024 6984454 -11710 13948890 13621.9629 308443.09823296947 308593.8156122219
-626923679 1024 9723.40270 -9778.95135 10541.05247 10.293996553 5742.091453325366 5744.897264122336 1024 11646 -11712 12641 12.3447 6877.306686989158 6880.6672084147185
-6981 2 -515.62107 -515.62107 -1031.24214 -515.621070000 0.0 0.0 3 6984454 -618 6983218 2327739.3333 3292794.518850853 4032833.1995089175
-762 1 1531.21941 1531.21941 1531.21941 1531.219410000 0.0 NULL 2 6984454 1834 6986288 3493144.0000 3491310.0 4937457.95244881
-NULL 3072 9318.43514 -4298.15135 5018444.11392 1633.608110000 5695.483083909642 5696.410309489072 3072 11161 -5148 6010880 1956.6667 6821.647911041892 6822.758476439734
+-3728 5 -515.62107 -3367.65176 -13986.22811 -2797.24562200000000000000000000000 1140.8122759999992 1275.466899351125 6 6984454 -4033 6967704 1161284.0000 2604201.0914565204 2852759.364140621
+-563 2 -515.62107 -3367.65176 -3883.27283 -1941.63641500000000000000000000000 1426.0153450000003 2016.6902410511484 2 -618 -4033 -4651 -2325.5000 1707.5 2414.7696577520596
+253665376 1024 9767.00541 -9779.54865 -347484.08192 -339.33992375000000000000000000000 5708.956347957812 5711.745967644425 1024 11698 -11713 -416183 -406.4287 6837.6426468206855 6840.983786842613
+528534767 1022 9777.75676 -9777.15946 -16711.67771 -16.35193513698630136986301369863 5555.762110793133 5558.482190324906 1024 6984454 -11710 13948890 13621.9629 308443.0982329696 308593.815612222
+626923679 1024 9723.40270 -9778.95135 10541.05247 10.29399655273437500000000000000 5742.091453325365 5744.897264122335 1024 11646 -11712 12641 12.3447 6877.306686989158 6880.6672084147185
+6981 2 -515.62107 -515.62107 -1031.24214 -515.62107000000000000000000000000 0.0 0.0 3 6984454 -618 6983218 2327739.3333 3292794.518850853 4032833.1995089175
+762 1 1531.21941 1531.21941 1531.21941 1531.21941000000000000000000000000 0.0 NULL 2 6984454 1834 6986288 3493144.0000 3491310.0 4937457.95244881
+NULL 3072 9318.43514 -4298.15135 5018444.11392 1633.60811000000000000000000000000 5695.4830839098695 5696.410309489299 3072 11161 -5148 6010880 1956.6667 6821.647911041892 6822.758476439734
PREHOOK: query: SELECT SUM(HASH(*))
FROM (SELECT cint,
COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1),
@@ -755,4 +815,4 @@ FROM (SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby_small
#### A masked pattern was here ####
-91757235680
+96966670826
http://git-wip-us.apache.org/repos/asf/hive/blob/5cb8867b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
index ec73876..c3201bf 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
@@ -403,18 +403,18 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ctinyint (type: tinyint)
- outputColumnNames: ctinyint
+ expressions: ctinyint (type: tinyint), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(ctinyint), variance(ctinyint), var_pop(ctinyint), var_samp(ctinyint), std(ctinyint), stddev(ctinyint), stddev_pop(ctinyint), stddev_samp(ctinyint)
+ aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: double), _col3 (type: double)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -426,22 +426,28 @@ STAGE PLANS:
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "variance" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFVarianceEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
+ aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (_col0 / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col3, _col4, _col7
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col3 (type: double), _col4 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
Reduce Vectorization:
@@ -452,12 +458,12 @@ STAGE PLANS:
vectorized: true
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
+ expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -497,7 +503,7 @@ ORDER BY c1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--4.344925324321378 1158.3003004768184 1158.3003004768184 1158.4265870337827 34.033811136527426 34.033811136527426 34.033811136527426 34.03566639620536
+-4.344925324321378 1158.3003004768175 1158.3003004768175 1158.426587033782 34.03381113652741 34.03381113652741 34.03381113652741 34.03566639620535
PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT MIN(cbigint) as c1,
MAX(cbigint),
@@ -903,18 +909,18 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cbigint (type: bigint)
- outputColumnNames: cbigint
+ expressions: cbigint (type: bigint), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cbigint), variance(cbigint), var_pop(cbigint), var_samp(cbigint), std(cbigint), stddev(cbigint), stddev_pop(cbigint), stddev_samp(cbigint)
+ aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: double), _col3 (type: double)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -926,22 +932,28 @@ STAGE PLANS:
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "variance" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFVarianceEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
+ aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (_col0 / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col3, _col4, _col7
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col3 (type: double), _col4 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
Reduce Vectorization:
@@ -952,12 +964,12 @@ STAGE PLANS:
vectorized: true
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
+ expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -997,7 +1009,7 @@ ORDER BY c1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--1.8515862077935246E8 2.07689300543081907E18 2.07689300543081907E18 2.07711944383088768E18 1.441142951074188E9 1.441142951074188E9 1.441142951074188E9 1.4412215110214279E9
+-1.8515862077935246E8 2.07689300543070106E18 2.07689300543070106E18 2.07711944383076992E18 1.441142951074147E9 1.441142951074147E9 1.441142951074147E9 1.441221511021387E9
PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT MIN(cfloat) as c1,
MAX(cfloat),
@@ -1403,18 +1415,18 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cfloat (type: float)
- outputColumnNames: cfloat
+ expressions: cfloat (type: float), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cfloat), variance(cfloat), var_pop(cfloat), var_samp(cfloat), std(cfloat), stddev(cfloat), stddev_pop(cfloat), stddev_samp(cfloat)
+ aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:float>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: double), _col1 (type: bigint), _col2 (type: double), _col3 (type: double)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -1426,22 +1438,28 @@ STAGE PLANS:
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "variance" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFVarianceEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
+ aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (_col0 / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / _col1) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col3, _col4, _col7
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col3 (type: double), _col4 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
Reduce Vectorization:
@@ -1452,12 +1470,12 @@ STAGE PLANS:
vectorized: true
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
+ expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1497,7 +1515,7 @@ ORDER BY c1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--4.303895780321011 1163.8972588604984 1163.8972588604984 1164.0241556397025 34.115938487171924 34.115938487171924 34.115938487171924 34.11779822379666
+-4.303895780321011 1163.8972588605056 1163.8972588605056 1164.0241556397098 34.11593848717203 34.11593848717203 34.11593848717203 34.11779822379677
WARNING: Comparing a bigint and a double may result in a loss of precision.
PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT AVG(cbigint),
@@ -1584,25 +1602,26 @@ STAGE PLANS:
predicate: (((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (UDFToInteger(ctinyint) = 3569)) or (CAST( cint AS decimal(13,3)) <> 79.553) or (UDFToDouble(cbigint) < cdouble) or (cstring2 like '%b%')) (type: boolean)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ctinyint (type: tinyint), cbigint (type: bigint), cfloat (type: float)
- outputColumnNames: ctinyint, cbigint, cfloat
+ expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [0, 3, 4]
+ projectedOutputColumnNums: [3, 4, 0, 14, 17]
+ selectExpressions: CastLongToDouble(col 3:bigint) -> 14:double, DoubleColMultiplyDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 15:double, CastLongToDouble(col 3:bigint) -> 16:double) -> 17:double
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cbigint), stddev_pop(cbigint), var_samp(cbigint), count(), sum(cfloat), min(ctinyint)
+ aggregations: sum(_col0), count(_col0), sum(_col4), sum(_col3), count(), sum(_col1), min(_col2)
Group By Vectorization:
- aggregators: VectorUDAFAvgLong(col 3:bigint) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFVarLong(col 3:bigint) -> struct<count:bigint,sum:double,variance:double> aggregation: stddev_pop, VectorUDAFVarLong(col 3:bigint) -> struct<count:bigint,sum:double,variance:double> aggregation: var_samp, VectorUDAFCountStar(*) -> bigint, VectorUDAFSumDouble(col 4:float) -> double, VectorUDAFMinLong(col 0:tinyint) -> tinyint
+ aggregators: VectorUDAFSumLong(col 3:bigint) -> bigint, VectorUDAFCount(col 3:bigint) -> bigint, VectorUDAFSumDouble(col 17:double) -> double, VectorUDAFSumDouble(col 14:double) -> double, VectorUDAFCountStar(*) -> bigint, VectorUDAFSumDouble(col 4:float) -> double, VectorUDAFMinLong(col 0:tinyint) -> tinyint
className: VectorGroupByOperator
groupByMode: HASH
native: false
vectorProcessingMode: HASH
- projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6]
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 1 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Reduce Sink Vectorization:
@@ -1610,9 +1629,9 @@ STAGE PLANS:
keyColumnNums: []
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [0, 1, 2, 3, 4, 5]
- Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: bigint), _col4 (type: double), _col5 (type: tinyint)
+ valueColumnNums: [0, 1, 2, 3, 4, 5, 6]
+ Statistics: Num rows: 1 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: tinyint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -1628,26 +1647,50 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3, 4, 5, 7, 11]
dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
partitionColumnCount: 0
- scratchColumnTypeNames: [decimal(13,3), double]
+ scratchColumnTypeNames: [decimal(13,3), double, double, double, double]
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ reduceColumnNullOrder:
+ reduceColumnSortOrder:
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 7
+ dataColumns: VALUE._col0:bigint, VALUE._col1:bigint, VALUE._col2:double, VALUE._col3:double, VALUE._col4:bigint, VALUE._col5:double, VALUE._col6:tinyint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
Reduce Operator Tree:
Group By Operator
- aggregations: avg(VALUE._col0), stddev_pop(VALUE._col1), var_samp(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), min(VALUE._col5)
+ aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), sum(VALUE._col5), min(VALUE._col6)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0:bigint) -> bigint, VectorUDAFCountMerge(col 1:bigint) -> bigint, VectorUDAFSumDouble(col 2:double) -> double, VectorUDAFSumDouble(col 3:double) -> double, VectorUDAFCountMerge(col 4:bigint) -> bigint, VectorUDAFSumDouble(col 5:double) -> double, VectorUDAFMinLong(col 6:tinyint) -> tinyint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ native: false
+ vectorProcessingMode: GLOBAL
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6]
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 1 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), (- _col0) (type: double), (-6432.0D + _col0) (type: double), _col1 (type: double), (- (-6432.0D + _col0)) (type: double), ((- (-6432.0D + _col0)) + (-6432.0D + _col0)) (type: double), _col2 (type: double), (- (-6432.0D + _col0)) (type: double), (-6432.0D + (- (-6432.0D + _col0))) (type: double), (- (-6432.0D + _col0)) (type: double), ((- (-6432.0D + _col0)) / (- (-6432.0D + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432.0D + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
+ expressions: (_col0 / _col1) (type: double), (- (_col0 / _col1)) (type: double), (-6432.0D + (_col0 / _col1)) (type: double), power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5) (type: double), (- (-6432.0D + (_col0 / _col1))) (type: double), ((- (-6432.0D + (_col0 / _col1))) + (-6432.0D + (_col0 / _col1))) (type: double), ((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END) (type: double), (- (-6432.0D + (_col0 / _col1))) (type: double), (-6432.0D + (- (-6432.0D + (_col0 / _col1)))) (type: double), (- (-6432.0D + (_col0 / _col1))) (type: double), ((- (-6432.0D + (_col0 / _col1))) / (- (-6432.0D + (_col0 / _col1)))) (type: double), _col4 (type: bigint), _col5 (type: double), (((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L)) THEN (null) ELSE ((_col1 - 1)) END) % power(((_col2 - ((_col3 * _col3) / _col1)) / _col1), 0.5)) (type: double), (- ((_col2 - ((_col3 * _col3) / _col1)) / CASE WHEN ((_col1 = 1L
)) THEN (null) ELSE ((_col1 - 1)) END)) (type: double), ((- (-6432.0D + (_col0 / _col1))) * (- (_col0 / _col1))) (type: double), _col6 (type: tinyint), (- _col6) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
- Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [7, 9, 10, 8, 11, 13, 14, 12, 19, 18, 22, 4, 5, 25, 20, 28, 6, 27]
+ selectExpressions: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 7:double, DoubleColUnaryMinus(col 8:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 8:double) -> 9:double, DoubleScalarAddDoubleColumn(val -6432.0, col 8:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 8:double) -> 10:double, FuncPowerDoubleToDouble(col 11:double)(children: DoubleColDivideLongColumn(col 8:double, col 1:bigint)(children: DoubleColSubtractDoubleColumn(col 2:double, col 11:double)(children: DoubleColDivideLongColumn(col 8:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 3:double, col 3:double) -> 8:double) -> 11:double) -> 8:double) -> 11:double) -> 8:double, DoubleColUnaryMinus(col 12:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 11:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 11:double) -> 12:double) -> 11:double, DoubleColAddDoubleColumn(col 12:double, col 14:do
uble)(children: DoubleColUnaryMinus(col 13:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 12:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 12:double) -> 13:double) -> 12:double, DoubleScalarAddDoubleColumn(val -6432.0, col 13:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 13:double) -> 14:double) -> 13:double, DoubleColDivideLongColumn(col 12:double, col 17:bigint)(children: DoubleColSubtractDoubleColumn(col 2:double, col 14:double)(children: DoubleColDivideLongColumn(col 12:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 3:double, col 3:double) -> 12:double) -> 14:double) -> 12:double, IfExprNullCondExpr(col 15:boolean, null, col 16:bigint)(children: LongColEqualLongScalar(col 1:bigint, val 1) -> 15:boolean, LongColSubtractLongScalar(col 1:bigint, val 1) -> 16:bigint) -> 17:bigint) -> 14:double, DoubleColUnaryMinus(col 18:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 12:double)(c
hildren: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 12:double) -> 18:double) -> 12:double, DoubleScalarAddDoubleColumn(val -6432.0, col 18:double)(children: DoubleColUnaryMinus(col 19:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 18:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 18:double) -> 19:double) -> 18:double) -> 19:double, DoubleColUnaryMinus(col 20:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 18:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 18:double) -> 20:double) -> 18:double, DoubleColDivideDoubleColumn(col 20:double, col 21:double)(children: DoubleColUnaryMinus(col 21:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 20:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 20:double) -> 21:double) -> 20:double, DoubleColUnaryMinus(col 22:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 21:double)(children: LongColDivideLongCol
umn(col 0:bigint, col 1:bigint) -> 21:double) -> 22:double) -> 21:double) -> 22:double, DoubleColModuloDoubleColumn(col 21:double, col 20:double)(children: DoubleColDivideLongColumn(col 20:double, col 24:bigint)(children: DoubleColSubtractDoubleColumn(col 2:double, col 21:double)(children: DoubleColDivideLongColumn(col 20:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 3:double, col 3:double) -> 20:double) -> 21:double) -> 20:double, IfExprNullCondExpr(col 17:boolean, null, col 23:bigint)(children: LongColEqualLongScalar(col 1:bigint, val 1) -> 17:boolean, LongColSubtractLongScalar(col 1:bigint, val 1) -> 23:bigint) -> 24:bigint) -> 21:double, FuncPowerDoubleToDouble(col 25:double)(children: DoubleColDivideLongColumn(col 20:double, col 1:bigint)(children: DoubleColSubtractDoubleColumn(col 2:double, col 25:double)(children: DoubleColDivideLongColumn(col 20:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 3:double, col 3:double) -> 20:double) -> 25:d
ouble) -> 20:double) -> 25:double) -> 20:double) -> 25:double, DoubleColUnaryMinus(col 21:double)(children: DoubleColDivideLongColumn(col 20:double, col 27:bigint)(children: DoubleColSubtractDoubleColumn(col 2:double, col 21:double)(children: DoubleColDivideLongColumn(col 20:double, col 1:bigint)(children: DoubleColMultiplyDoubleColumn(col 3:double, col 3:double) -> 20:double) -> 21:double) -> 20:double, IfExprNullCondExpr(col 24:boolean, null, col 26:bigint)(children: LongColEqualLongScalar(col 1:bigint, val 1) -> 24:boolean, LongColSubtractLongScalar(col 1:bigint, val 1) -> 26:bigint) -> 27:bigint) -> 21:double) -> 20:double, DoubleColMultiplyDoubleColumn(col 21:double, col 29:double)(children: DoubleColUnaryMinus(col 28:double)(children: DoubleScalarAddDoubleColumn(val -6432.0, col 21:double)(children: LongColDivideLongColumn(col 0:bigint, col 1:bigint) -> 21:double) -> 28:double) -> 21:double, DoubleColUnaryMinus(col 28:double)(children: LongColDivideLongColumn(col 0:bigint, col
1:bigint) -> 28:double) -> 29:double) -> 28:double, LongColUnaryMinus(col 6:tinyint) -> 27:tinyint
+ Statistics: Num rows: 1 Data size: 52 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 52 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1716,7 +1759,7 @@ WHERE (((cstring2 LIKE '%b%')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--3.875652215945533E8 3.875652215945533E8 -3.875716535945533E8 1.436387455459401E9 3.875716535945533E8 0.0 2.06347151720204902E18 3.875716535945533E8 3.875652215945533E8 3.875716535945533E8 1.0 10934 -37224.52399241924 1.0517370547117279E9 -2.06347151720204902E18 1.5020929380914048E17 -64 64
+-3.875652215945533E8 3.875652215945533E8 -3.875716535945533E8 1.4363874554593627E9 3.875716535945533E8 0.0 2.0634715172019392E18 3.875716535945533E8 3.875652215945533E8 3.875716535945533E8 1.0 10934 -37224.52399241924 1.0516820315185745E9 -2.0634715172019392E18 1.5020929380914048E17 -64 64
PREHOOK: query: EXPLAIN extended
select count(*) from alltypesorc
where (((cstring1 LIKE 'a%') or ((cstring1 like 'b%') or (cstring1 like 'c%'))) or