You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/03/22 16:57:29 UTC
[06/34] hive git commit: HIVE-18979: Enable
AggregateReduceFunctionsRule from Calcite (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/5cb8867b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
index 01eb4b4..830d0f8 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
@@ -171,21 +171,21 @@ STAGE PLANS:
alias: alltypes_parquet
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
- outputColumnNames: cint, ctinyint, csmallint, cfloat, cdouble, cstring1
+ expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble)
- keys: ctinyint (type: tinyint)
+ aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
+ keys: _col0 (type: tinyint)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -197,25 +197,31 @@ STAGE PLANS:
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
+ aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), count(VALUE._col7)
keys: KEY._col0 (type: tinyint)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -245,36 +251,36 @@ POSTHOOK: query: select ctinyint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypes_parquet
#### A masked pattern was here ####
--1 626923679 -15441 36 -1.0486250072717667 8786.246963933321
+-1 626923679 -15441 36 -1.0486250072717667 8786.246963933323
-10 626923679 -15384 28 -10.0 8850.451610567823
--11 626923679 -15659 32 -11.0 10453.738567408038
--12 626923679 -16373 22 -12.0 10173.15707541171
--13 626923679 -15446 30 -13.0 8907.942987576693
+-11 626923679 -15659 32 -11.0 10453.73856740804
+-12 626923679 -16373 22 -12.0 10173.157075411711
+-13 626923679 -15446 30 -13.0 8907.942987576691
-14 626923679 -13884 22 -14.0 10125.818731386042
--15 626923679 -16036 24 -15.0 9450.506254395024
+-15 626923679 -16036 24 -15.0 9450.506254395026
-16 626923679 -15154 21 -16.0 8884.207393686478
-17 626923679 -15922 19 -17.0 9944.104273894172
-18 626923679 -14863 24 -18.0 9638.430684071413
-19 626923679 -15935 25 -19.0 9967.22240685782
-2 626923679 -16277 20 -2.0 10800.090249507177
--20 626923679 -16126 24 -20.0 9868.92268080106
+-20 626923679 -16126 24 -20.0 9868.922680801063
-21 626923679 -16017 27 -21.0 9480.349236669877
-22 626923679 -14701 22 -22.0 8809.230165774987
-23 626923679 -16355 36 -23.345263230173213 9401.831290253447
-24 626923679 -16311 26 -24.0 9386.736402961187
--25 626923679 -15862 24 -25.0 9778.256724727018
--26 626923679 -15686 15 -26.0 10874.523900405318
+-25 626923679 -15862 24 -25.0 9778.25672472702
+-26 626923679 -15686 15 -26.0 10874.52390040532
-27 626923679 -14984 20 -27.0 8465.29660255097
-28 626923679 -15813 20 -28.0 9616.869413270924
--29 626923679 -14747 26 -29.0 9052.945656011721
+-29 626923679 -14747 26 -29.0 9052.945656011723
-3 626923679 -13632 16 -3.0 8836.215573422822
--30 626923679 -14863 23 -30.0 9193.941914019653
+-30 626923679 -14863 23 -30.0 9193.941914019651
-31 626923679 -15915 22 -31.0 9187.596784112568
--32 626923679 -15866 25 -32.0 9535.546396775915
+-32 626923679 -15866 25 -32.0 9535.546396775917
-33 626923679 -12779 21 -33.0 8854.331159704514
-34 626923679 -15450 29 -34.0 8708.243526705026
-35 626923679 -16059 23 -35.0 10136.580492864763
--36 626923679 -16208 23 -36.0 8773.547684436919
+-36 626923679 -16208 23 -36.0 8773.54768443692
-37 626923679 -14780 17 -37.0 10368.905538788269
-38 626923679 -14914 28 -38.0 8767.375358291503
-39 626923679 -15612 19 -39.0 9765.551806305297
@@ -287,37 +293,37 @@ POSTHOOK: Input: default@alltypes_parquet
-45 626923679 -15027 21 -45.0 8567.489593562543
-46 626923679 -12427 21 -46.0 9182.943188188632
-47 626923679 -16096 19 -47.0 9011.009178780589
--48 626923679 -15462 26 -48.0 9913.883371354861
+-48 626923679 -15462 26 -48.0 9913.883371354863
-49 626923679 -14831 23 -49.0 9894.429191738676
-5 626923679 -15780 24 -5.0 10599.227726422314
-50 626923679 -14320 27 -50.0 8548.827748002343
-51 1073680599 -15734 1028 -51.0 9531.569305177045
-52 626923679 -16369 30 -52.0 8625.06871423408
--53 626923679 -15445 19 -53.0 9387.739325499799
--54 626923679 -14815 23 -54.0 9614.154026896626
+-53 626923679 -15445 19 -53.0 9387.7393254998
+-54 626923679 -14815 23 -54.0 9614.154026896624
-55 626923679 -13381 26 -55.0 9157.562103946742
--56 626923679 -11999 33 -56.0 9490.842152672341
--57 626923679 -14893 32 -57.0 8572.083461570477
--58 626923679 -15169 20 -58.0 9549.096672008198
--59 626923679 -15789 28 -59.0 9829.790704244733
+-56 626923679 -11999 33 -56.0 9490.84215267234
+-57 626923679 -14893 32 -57.0 8572.083461570479
+-58 626923679 -15169 20 -58.0 9549.096672008196
+-59 626923679 -15789 28 -59.0 9829.790704244735
-6 626923679 -15980 30 -6.0 10262.829252317424
-60 626923679 -15792 24 -60.0 9892.656196775464
--61 626923679 -15142 22 -61.0 9357.236187870849
--62 626923679 -15992 24 -62.0 9004.593091474135
+-61 626923679 -15142 22 -61.0 9357.23618787085
+-62 626923679 -15992 24 -62.0 9004.593091474137
-63 626923679 -12516 16 -63.0 9263.605837223322
-64 626923679 -15920 21 -64.0 9254.456539277186
--7 626923679 -14584 23 -7.0 9946.605446407746
--8 626923679 -14678 18 -8.0 9976.831992670684
+-7 626923679 -14584 23 -7.0 9946.605446407748
+-8 626923679 -14678 18 -8.0 9976.831992670686
-9 626923679 -15329 31 -9.0 8999.391457373968
0 626923679 -14254 24 0.0 10057.5018088718
-1 626923679 -14610 30 1.0 10016.486277900643
-10 626923679 -15887 26 10.0 9104.820520135108
+1 626923679 -14610 30 1.0 10016.486277900645
+10 626923679 -15887 26 10.0 9104.82052013511
11 1072654057 -14696 1035 11.0 9531.018991371746
12 626923679 -14642 18 12.0 9696.038286378725
13 626923679 -14771 26 13.0 8128.265919972384
14 626923679 -13367 28 14.0 9074.674998750581
-15 626923679 -16339 28 15.0 9770.473400901916
-16 626923679 -14001 26 16.0 10130.883606275334
+15 626923679 -16339 28 15.0 9770.473400901918
+16 626923679 -14001 26 16.0 10130.883606275338
17 626923679 -16109 22 16.73235294865627 1353416.3383574807
18 626923679 -15779 21 18.0 10820.004053788869
19 626923679 -16049 21 19.0 9423.560227007669
@@ -328,28 +334,28 @@ POSTHOOK: Input: default@alltypes_parquet
23 626923679 -15514 24 23.0 8542.419116415425
24 626923679 -15086 24 24.0 9661.203790645088
25 626923679 -11349 23 25.0 8888.959012093468
-26 626923679 -14516 29 26.0 9123.125508880432
-27 626923679 -14965 24 27.0 9802.871860196345
+26 626923679 -14516 29 26.0 9123.125508880434
+27 626923679 -14965 24 27.0 9802.871860196343
28 626923679 -14455 20 28.0 9283.289383115296
29 626923679 -15892 16 29.0 9874.046501817154
-3 626923679 -16339 30 3.0 10483.526375885149
-30 626923679 -14111 27 30.0 10066.520234676527
+3 626923679 -16339 30 3.0 10483.526375885147
+30 626923679 -14111 27 30.0 10066.520234676529
31 626923679 -15960 24 31.0 10427.970184550613
32 626923679 -14044 24 32.0 8376.464579403413
-33 626923679 -14642 29 40.61776386607777 1304429.5939037625
-34 626923679 -15059 28 34.0 8756.731536033676
+33 626923679 -14642 29 40.61776386607777 1304429.593903763
+34 626923679 -15059 28 34.0 8756.731536033674
35 626923679 -16153 27 35.0 10351.008404963042
36 626923679 -15912 20 36.0 9475.257975138164
37 626923679 -12081 24 37.0 9017.860034890362
38 626923679 -15248 29 38.0 9900.256257785535
-39 626923679 -14887 28 39.0 10513.343644635232
-4 626923679 -15999 29 4.0 9516.189702058042
+39 626923679 -14887 28 39.0 10513.343644635233
+4 626923679 -15999 29 4.0 9516.189702058044
40 626923679 -15861 22 40.0 9283.318678549174
-41 626923679 -13480 21 41.0 9016.291129937847
+41 626923679 -13480 21 41.0 9016.291129937848
42 626923679 -15834 28 42.0 10318.01399719996
43 626923679 -15703 28 43.0 8757.796089055722
44 626923679 -11185 16 44.0 9425.076634933797
-45 626923679 -15228 18 45.0 9459.968668643689
+45 626923679 -15228 18 45.0 9459.968668643687
46 626923679 -15187 22 46.0 9685.908173160062
47 626923679 -16324 22 47.0 9822.220821743611
48 626923679 -16372 29 48.0 10079.286173063345
@@ -360,18 +366,18 @@ POSTHOOK: Input: default@alltypes_parquet
52 626923679 -15450 20 52.0 9261.723648435052
53 626923679 -16217 30 53.0 9895.247408969733
54 626923679 -15245 16 54.0 9789.50878424882
-55 626923679 -15887 21 55.0 9826.38569192808
+55 626923679 -15887 21 55.0 9826.385691928082
56 626923679 -12631 21 56.0 8860.917133763547
57 626923679 -15620 25 57.0 9413.99393840875
58 626923679 -13627 20 58.0 9083.529665947459
-59 626923679 -16076 17 59.0 10117.44967077967
-6 626923679 -15948 30 6.0 9644.247255286113
+59 626923679 -16076 17 59.0 10117.449670779672
+6 626923679 -15948 30 6.0 9644.247255286115
60 626923679 -13606 23 60.0 8346.267436552042
-61 626923679 -15894 29 61.0 8785.714950987198
-62 626923679 -14307 17 62.0 9491.752726667326
+61 626923679 -15894 29 61.0 8785.7149509872
+62 626923679 -14307 17 62.0 9491.752726667324
7 626923679 -15839 25 7.0 10077.151640330823
8 1070764888 -15778 1034 8.0 9562.355155774725
-9 626923679 -13629 25 9.0 10157.217948808622
+9 626923679 -13629 25 9.0 10157.21794880862
NULL 1073418988 -16379 3115 NULL 305051.4870777435
PREHOOK: query: explain vectorization select *
from alltypes_parquet
@@ -492,45 +498,51 @@ STAGE PLANS:
alias: alltypes_parquet
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
- outputColumnNames: cint, ctinyint, csmallint, cfloat, cdouble, cstring1
+ expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble)
- keys: ctinyint (type: tinyint)
+ aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
+ keys: _col0 (type: tinyint)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Map Vectorization:
enabled: false
enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false
inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
+ aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), count(VALUE._col7)
keys: KEY._col0 (type: tinyint)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -560,36 +572,36 @@ POSTHOOK: query: select ctinyint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypes_parquet
#### A masked pattern was here ####
--1 626923679 -15441 36 -1.0486250072717667 8786.246963933321
+-1 626923679 -15441 36 -1.0486250072717667 8786.246963933323
-10 626923679 -15384 28 -10.0 8850.451610567823
--11 626923679 -15659 32 -11.0 10453.738567408038
--12 626923679 -16373 22 -12.0 10173.15707541171
--13 626923679 -15446 30 -13.0 8907.942987576693
+-11 626923679 -15659 32 -11.0 10453.73856740804
+-12 626923679 -16373 22 -12.0 10173.157075411711
+-13 626923679 -15446 30 -13.0 8907.942987576691
-14 626923679 -13884 22 -14.0 10125.818731386042
--15 626923679 -16036 24 -15.0 9450.506254395024
+-15 626923679 -16036 24 -15.0 9450.506254395026
-16 626923679 -15154 21 -16.0 8884.207393686478
-17 626923679 -15922 19 -17.0 9944.104273894172
-18 626923679 -14863 24 -18.0 9638.430684071413
-19 626923679 -15935 25 -19.0 9967.22240685782
-2 626923679 -16277 20 -2.0 10800.090249507177
--20 626923679 -16126 24 -20.0 9868.92268080106
+-20 626923679 -16126 24 -20.0 9868.922680801063
-21 626923679 -16017 27 -21.0 9480.349236669877
-22 626923679 -14701 22 -22.0 8809.230165774987
-23 626923679 -16355 36 -23.345263230173213 9401.831290253447
-24 626923679 -16311 26 -24.0 9386.736402961187
--25 626923679 -15862 24 -25.0 9778.256724727018
--26 626923679 -15686 15 -26.0 10874.523900405318
+-25 626923679 -15862 24 -25.0 9778.25672472702
+-26 626923679 -15686 15 -26.0 10874.52390040532
-27 626923679 -14984 20 -27.0 8465.29660255097
-28 626923679 -15813 20 -28.0 9616.869413270924
--29 626923679 -14747 26 -29.0 9052.945656011721
+-29 626923679 -14747 26 -29.0 9052.945656011723
-3 626923679 -13632 16 -3.0 8836.215573422822
--30 626923679 -14863 23 -30.0 9193.941914019653
+-30 626923679 -14863 23 -30.0 9193.941914019651
-31 626923679 -15915 22 -31.0 9187.596784112568
--32 626923679 -15866 25 -32.0 9535.546396775915
+-32 626923679 -15866 25 -32.0 9535.546396775917
-33 626923679 -12779 21 -33.0 8854.331159704514
-34 626923679 -15450 29 -34.0 8708.243526705026
-35 626923679 -16059 23 -35.0 10136.580492864763
--36 626923679 -16208 23 -36.0 8773.547684436919
+-36 626923679 -16208 23 -36.0 8773.54768443692
-37 626923679 -14780 17 -37.0 10368.905538788269
-38 626923679 -14914 28 -38.0 8767.375358291503
-39 626923679 -15612 19 -39.0 9765.551806305297
@@ -602,37 +614,37 @@ POSTHOOK: Input: default@alltypes_parquet
-45 626923679 -15027 21 -45.0 8567.489593562543
-46 626923679 -12427 21 -46.0 9182.943188188632
-47 626923679 -16096 19 -47.0 9011.009178780589
--48 626923679 -15462 26 -48.0 9913.883371354861
+-48 626923679 -15462 26 -48.0 9913.883371354863
-49 626923679 -14831 23 -49.0 9894.429191738676
-5 626923679 -15780 24 -5.0 10599.227726422314
-50 626923679 -14320 27 -50.0 8548.827748002343
-51 1073680599 -15734 1028 -51.0 9531.569305177045
-52 626923679 -16369 30 -52.0 8625.06871423408
--53 626923679 -15445 19 -53.0 9387.739325499799
--54 626923679 -14815 23 -54.0 9614.154026896626
+-53 626923679 -15445 19 -53.0 9387.7393254998
+-54 626923679 -14815 23 -54.0 9614.154026896624
-55 626923679 -13381 26 -55.0 9157.562103946742
--56 626923679 -11999 33 -56.0 9490.842152672341
--57 626923679 -14893 32 -57.0 8572.083461570477
--58 626923679 -15169 20 -58.0 9549.096672008198
--59 626923679 -15789 28 -59.0 9829.790704244733
+-56 626923679 -11999 33 -56.0 9490.84215267234
+-57 626923679 -14893 32 -57.0 8572.083461570479
+-58 626923679 -15169 20 -58.0 9549.096672008196
+-59 626923679 -15789 28 -59.0 9829.790704244735
-6 626923679 -15980 30 -6.0 10262.829252317424
-60 626923679 -15792 24 -60.0 9892.656196775464
--61 626923679 -15142 22 -61.0 9357.236187870849
--62 626923679 -15992 24 -62.0 9004.593091474135
+-61 626923679 -15142 22 -61.0 9357.23618787085
+-62 626923679 -15992 24 -62.0 9004.593091474137
-63 626923679 -12516 16 -63.0 9263.605837223322
-64 626923679 -15920 21 -64.0 9254.456539277186
--7 626923679 -14584 23 -7.0 9946.605446407746
--8 626923679 -14678 18 -8.0 9976.831992670684
+-7 626923679 -14584 23 -7.0 9946.605446407748
+-8 626923679 -14678 18 -8.0 9976.831992670686
-9 626923679 -15329 31 -9.0 8999.391457373968
0 626923679 -14254 24 0.0 10057.5018088718
-1 626923679 -14610 30 1.0 10016.486277900643
-10 626923679 -15887 26 10.0 9104.820520135108
+1 626923679 -14610 30 1.0 10016.486277900645
+10 626923679 -15887 26 10.0 9104.82052013511
11 1072654057 -14696 1035 11.0 9531.018991371746
12 626923679 -14642 18 12.0 9696.038286378725
13 626923679 -14771 26 13.0 8128.265919972384
14 626923679 -13367 28 14.0 9074.674998750581
-15 626923679 -16339 28 15.0 9770.473400901916
-16 626923679 -14001 26 16.0 10130.883606275334
+15 626923679 -16339 28 15.0 9770.473400901918
+16 626923679 -14001 26 16.0 10130.883606275338
17 626923679 -16109 22 16.73235294865627 1353416.3383574807
18 626923679 -15779 21 18.0 10820.004053788869
19 626923679 -16049 21 19.0 9423.560227007669
@@ -643,28 +655,28 @@ POSTHOOK: Input: default@alltypes_parquet
23 626923679 -15514 24 23.0 8542.419116415425
24 626923679 -15086 24 24.0 9661.203790645088
25 626923679 -11349 23 25.0 8888.959012093468
-26 626923679 -14516 29 26.0 9123.125508880432
-27 626923679 -14965 24 27.0 9802.871860196345
+26 626923679 -14516 29 26.0 9123.125508880434
+27 626923679 -14965 24 27.0 9802.871860196343
28 626923679 -14455 20 28.0 9283.289383115296
29 626923679 -15892 16 29.0 9874.046501817154
-3 626923679 -16339 30 3.0 10483.526375885149
-30 626923679 -14111 27 30.0 10066.520234676527
+3 626923679 -16339 30 3.0 10483.526375885147
+30 626923679 -14111 27 30.0 10066.520234676529
31 626923679 -15960 24 31.0 10427.970184550613
32 626923679 -14044 24 32.0 8376.464579403413
-33 626923679 -14642 29 40.61776386607777 1304429.5939037625
-34 626923679 -15059 28 34.0 8756.731536033676
+33 626923679 -14642 29 40.61776386607777 1304429.593903763
+34 626923679 -15059 28 34.0 8756.731536033674
35 626923679 -16153 27 35.0 10351.008404963042
36 626923679 -15912 20 36.0 9475.257975138164
37 626923679 -12081 24 37.0 9017.860034890362
38 626923679 -15248 29 38.0 9900.256257785535
-39 626923679 -14887 28 39.0 10513.343644635232
-4 626923679 -15999 29 4.0 9516.189702058042
+39 626923679 -14887 28 39.0 10513.343644635233
+4 626923679 -15999 29 4.0 9516.189702058044
40 626923679 -15861 22 40.0 9283.318678549174
-41 626923679 -13480 21 41.0 9016.291129937847
+41 626923679 -13480 21 41.0 9016.291129937848
42 626923679 -15834 28 42.0 10318.01399719996
43 626923679 -15703 28 43.0 8757.796089055722
44 626923679 -11185 16 44.0 9425.076634933797
-45 626923679 -15228 18 45.0 9459.968668643689
+45 626923679 -15228 18 45.0 9459.968668643687
46 626923679 -15187 22 46.0 9685.908173160062
47 626923679 -16324 22 47.0 9822.220821743611
48 626923679 -16372 29 48.0 10079.286173063345
@@ -675,18 +687,18 @@ POSTHOOK: Input: default@alltypes_parquet
52 626923679 -15450 20 52.0 9261.723648435052
53 626923679 -16217 30 53.0 9895.247408969733
54 626923679 -15245 16 54.0 9789.50878424882
-55 626923679 -15887 21 55.0 9826.38569192808
+55 626923679 -15887 21 55.0 9826.385691928082
56 626923679 -12631 21 56.0 8860.917133763547
57 626923679 -15620 25 57.0 9413.99393840875
58 626923679 -13627 20 58.0 9083.529665947459
-59 626923679 -16076 17 59.0 10117.44967077967
-6 626923679 -15948 30 6.0 9644.247255286113
+59 626923679 -16076 17 59.0 10117.449670779672
+6 626923679 -15948 30 6.0 9644.247255286115
60 626923679 -13606 23 60.0 8346.267436552042
-61 626923679 -15894 29 61.0 8785.714950987198
-62 626923679 -14307 17 62.0 9491.752726667326
+61 626923679 -15894 29 61.0 8785.7149509872
+62 626923679 -14307 17 62.0 9491.752726667324
7 626923679 -15839 25 7.0 10077.151640330823
8 1070764888 -15778 1034 8.0 9562.355155774725
-9 626923679 -13629 25 9.0 10157.217948808622
+9 626923679 -13629 25 9.0 10157.21794880862
NULL 1073418988 -16379 3115 NULL 305051.4870777435
PREHOOK: query: explain vectorization select *
from alltypes_parquet
@@ -813,21 +825,21 @@ STAGE PLANS:
alias: alltypes_parquet
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
- outputColumnNames: cint, ctinyint, csmallint, cfloat, cdouble, cstring1
+ expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble)
- keys: ctinyint (type: tinyint)
+ aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
+ keys: _col0 (type: tinyint)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -839,25 +851,31 @@ STAGE PLANS:
usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
+ aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), count(VALUE._col7)
keys: KEY._col0 (type: tinyint)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -887,36 +905,36 @@ POSTHOOK: query: select ctinyint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypes_parquet
#### A masked pattern was here ####
--1 626923679 -15441 36 -1.0486250072717667 8786.246963933321
+-1 626923679 -15441 36 -1.0486250072717667 8786.246963933323
-10 626923679 -15384 28 -10.0 8850.451610567823
--11 626923679 -15659 32 -11.0 10453.738567408038
--12 626923679 -16373 22 -12.0 10173.15707541171
--13 626923679 -15446 30 -13.0 8907.942987576693
+-11 626923679 -15659 32 -11.0 10453.73856740804
+-12 626923679 -16373 22 -12.0 10173.157075411711
+-13 626923679 -15446 30 -13.0 8907.942987576691
-14 626923679 -13884 22 -14.0 10125.818731386042
--15 626923679 -16036 24 -15.0 9450.506254395024
+-15 626923679 -16036 24 -15.0 9450.506254395026
-16 626923679 -15154 21 -16.0 8884.207393686478
-17 626923679 -15922 19 -17.0 9944.104273894172
-18 626923679 -14863 24 -18.0 9638.430684071413
-19 626923679 -15935 25 -19.0 9967.22240685782
-2 626923679 -16277 20 -2.0 10800.090249507177
--20 626923679 -16126 24 -20.0 9868.92268080106
+-20 626923679 -16126 24 -20.0 9868.922680801063
-21 626923679 -16017 27 -21.0 9480.349236669877
-22 626923679 -14701 22 -22.0 8809.230165774987
-23 626923679 -16355 36 -23.345263230173213 9401.831290253447
-24 626923679 -16311 26 -24.0 9386.736402961187
--25 626923679 -15862 24 -25.0 9778.256724727018
--26 626923679 -15686 15 -26.0 10874.523900405318
+-25 626923679 -15862 24 -25.0 9778.25672472702
+-26 626923679 -15686 15 -26.0 10874.52390040532
-27 626923679 -14984 20 -27.0 8465.29660255097
-28 626923679 -15813 20 -28.0 9616.869413270924
--29 626923679 -14747 26 -29.0 9052.945656011721
+-29 626923679 -14747 26 -29.0 9052.945656011723
-3 626923679 -13632 16 -3.0 8836.215573422822
--30 626923679 -14863 23 -30.0 9193.941914019653
+-30 626923679 -14863 23 -30.0 9193.941914019651
-31 626923679 -15915 22 -31.0 9187.596784112568
--32 626923679 -15866 25 -32.0 9535.546396775915
+-32 626923679 -15866 25 -32.0 9535.546396775917
-33 626923679 -12779 21 -33.0 8854.331159704514
-34 626923679 -15450 29 -34.0 8708.243526705026
-35 626923679 -16059 23 -35.0 10136.580492864763
--36 626923679 -16208 23 -36.0 8773.547684436919
+-36 626923679 -16208 23 -36.0 8773.54768443692
-37 626923679 -14780 17 -37.0 10368.905538788269
-38 626923679 -14914 28 -38.0 8767.375358291503
-39 626923679 -15612 19 -39.0 9765.551806305297
@@ -929,37 +947,37 @@ POSTHOOK: Input: default@alltypes_parquet
-45 626923679 -15027 21 -45.0 8567.489593562543
-46 626923679 -12427 21 -46.0 9182.943188188632
-47 626923679 -16096 19 -47.0 9011.009178780589
--48 626923679 -15462 26 -48.0 9913.883371354861
+-48 626923679 -15462 26 -48.0 9913.883371354863
-49 626923679 -14831 23 -49.0 9894.429191738676
-5 626923679 -15780 24 -5.0 10599.227726422314
-50 626923679 -14320 27 -50.0 8548.827748002343
-51 1073680599 -15734 1028 -51.0 9531.569305177045
-52 626923679 -16369 30 -52.0 8625.06871423408
--53 626923679 -15445 19 -53.0 9387.739325499799
--54 626923679 -14815 23 -54.0 9614.154026896626
+-53 626923679 -15445 19 -53.0 9387.7393254998
+-54 626923679 -14815 23 -54.0 9614.154026896624
-55 626923679 -13381 26 -55.0 9157.562103946742
--56 626923679 -11999 33 -56.0 9490.842152672341
--57 626923679 -14893 32 -57.0 8572.083461570477
--58 626923679 -15169 20 -58.0 9549.096672008198
--59 626923679 -15789 28 -59.0 9829.790704244733
+-56 626923679 -11999 33 -56.0 9490.84215267234
+-57 626923679 -14893 32 -57.0 8572.083461570479
+-58 626923679 -15169 20 -58.0 9549.096672008196
+-59 626923679 -15789 28 -59.0 9829.790704244735
-6 626923679 -15980 30 -6.0 10262.829252317424
-60 626923679 -15792 24 -60.0 9892.656196775464
--61 626923679 -15142 22 -61.0 9357.236187870849
--62 626923679 -15992 24 -62.0 9004.593091474135
+-61 626923679 -15142 22 -61.0 9357.23618787085
+-62 626923679 -15992 24 -62.0 9004.593091474137
-63 626923679 -12516 16 -63.0 9263.605837223322
-64 626923679 -15920 21 -64.0 9254.456539277186
--7 626923679 -14584 23 -7.0 9946.605446407746
--8 626923679 -14678 18 -8.0 9976.831992670684
+-7 626923679 -14584 23 -7.0 9946.605446407748
+-8 626923679 -14678 18 -8.0 9976.831992670686
-9 626923679 -15329 31 -9.0 8999.391457373968
0 626923679 -14254 24 0.0 10057.5018088718
-1 626923679 -14610 30 1.0 10016.486277900643
-10 626923679 -15887 26 10.0 9104.820520135108
+1 626923679 -14610 30 1.0 10016.486277900645
+10 626923679 -15887 26 10.0 9104.82052013511
11 1072654057 -14696 1035 11.0 9531.018991371746
12 626923679 -14642 18 12.0 9696.038286378725
13 626923679 -14771 26 13.0 8128.265919972384
14 626923679 -13367 28 14.0 9074.674998750581
-15 626923679 -16339 28 15.0 9770.473400901916
-16 626923679 -14001 26 16.0 10130.883606275334
+15 626923679 -16339 28 15.0 9770.473400901918
+16 626923679 -14001 26 16.0 10130.883606275338
17 626923679 -16109 22 16.73235294865627 1353416.3383574807
18 626923679 -15779 21 18.0 10820.004053788869
19 626923679 -16049 21 19.0 9423.560227007669
@@ -970,28 +988,28 @@ POSTHOOK: Input: default@alltypes_parquet
23 626923679 -15514 24 23.0 8542.419116415425
24 626923679 -15086 24 24.0 9661.203790645088
25 626923679 -11349 23 25.0 8888.959012093468
-26 626923679 -14516 29 26.0 9123.125508880432
-27 626923679 -14965 24 27.0 9802.871860196345
+26 626923679 -14516 29 26.0 9123.125508880434
+27 626923679 -14965 24 27.0 9802.871860196343
28 626923679 -14455 20 28.0 9283.289383115296
29 626923679 -15892 16 29.0 9874.046501817154
-3 626923679 -16339 30 3.0 10483.526375885149
-30 626923679 -14111 27 30.0 10066.520234676527
+3 626923679 -16339 30 3.0 10483.526375885147
+30 626923679 -14111 27 30.0 10066.520234676529
31 626923679 -15960 24 31.0 10427.970184550613
32 626923679 -14044 24 32.0 8376.464579403413
-33 626923679 -14642 29 40.61776386607777 1304429.5939037625
-34 626923679 -15059 28 34.0 8756.731536033676
+33 626923679 -14642 29 40.61776386607777 1304429.593903763
+34 626923679 -15059 28 34.0 8756.731536033674
35 626923679 -16153 27 35.0 10351.008404963042
36 626923679 -15912 20 36.0 9475.257975138164
37 626923679 -12081 24 37.0 9017.860034890362
38 626923679 -15248 29 38.0 9900.256257785535
-39 626923679 -14887 28 39.0 10513.343644635232
-4 626923679 -15999 29 4.0 9516.189702058042
+39 626923679 -14887 28 39.0 10513.343644635233
+4 626923679 -15999 29 4.0 9516.189702058044
40 626923679 -15861 22 40.0 9283.318678549174
-41 626923679 -13480 21 41.0 9016.291129937847
+41 626923679 -13480 21 41.0 9016.291129937848
42 626923679 -15834 28 42.0 10318.01399719996
43 626923679 -15703 28 43.0 8757.796089055722
44 626923679 -11185 16 44.0 9425.076634933797
-45 626923679 -15228 18 45.0 9459.968668643689
+45 626923679 -15228 18 45.0 9459.968668643687
46 626923679 -15187 22 46.0 9685.908173160062
47 626923679 -16324 22 47.0 9822.220821743611
48 626923679 -16372 29 48.0 10079.286173063345
@@ -1002,18 +1020,18 @@ POSTHOOK: Input: default@alltypes_parquet
52 626923679 -15450 20 52.0 9261.723648435052
53 626923679 -16217 30 53.0 9895.247408969733
54 626923679 -15245 16 54.0 9789.50878424882
-55 626923679 -15887 21 55.0 9826.38569192808
+55 626923679 -15887 21 55.0 9826.385691928082
56 626923679 -12631 21 56.0 8860.917133763547
57 626923679 -15620 25 57.0 9413.99393840875
58 626923679 -13627 20 58.0 9083.529665947459
-59 626923679 -16076 17 59.0 10117.44967077967
-6 626923679 -15948 30 6.0 9644.247255286113
+59 626923679 -16076 17 59.0 10117.449670779672
+6 626923679 -15948 30 6.0 9644.247255286115
60 626923679 -13606 23 60.0 8346.267436552042
-61 626923679 -15894 29 61.0 8785.714950987198
-62 626923679 -14307 17 62.0 9491.752726667326
+61 626923679 -15894 29 61.0 8785.7149509872
+62 626923679 -14307 17 62.0 9491.752726667324
7 626923679 -15839 25 7.0 10077.151640330823
8 1070764888 -15778 1034 8.0 9562.355155774725
-9 626923679 -13629 25 9.0 10157.217948808622
+9 626923679 -13629 25 9.0 10157.21794880862
NULL 1073418988 -16379 3115 NULL 305051.4870777435
PREHOOK: query: create table if not exists alltypes_orc (
cint int,
@@ -1182,45 +1200,51 @@ STAGE PLANS:
alias: alltypes_orc
Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
- outputColumnNames: cint, ctinyint, csmallint, cfloat, cdouble, cstring1
+ expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble)
- keys: ctinyint (type: tinyint)
+ aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
+ keys: _col0 (type: tinyint)
mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Map Vectorization:
enabled: false
enabledConditionsNotMet: hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.orc.OrcInputFormat IS false
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
Reducer 2
+ Execution mode: vectorized
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- notVectorizedReason: GROUPBY operator: Vector aggregation : "stddev_pop" for input type: "STRUCT" and output type: "DOUBLE" and mode: FINAL not supported for evaluator GenericUDAFStdEvaluator
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
+ aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), sum(VALUE._col3), count(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), count(VALUE._col7)
keys: KEY._col0 (type: tinyint)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1250,36 +1274,36 @@ POSTHOOK: query: select ctinyint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypes_orc
#### A masked pattern was here ####
--1 626923679 -15441 36 -1.0486250072717667 8786.246963933321
+-1 626923679 -15441 36 -1.0486250072717667 8786.246963933323
-10 626923679 -15384 28 -10.0 8850.451610567823
--11 626923679 -15659 32 -11.0 10453.738567408038
--12 626923679 -16373 22 -12.0 10173.15707541171
--13 626923679 -15446 30 -13.0 8907.942987576693
+-11 626923679 -15659 32 -11.0 10453.73856740804
+-12 626923679 -16373 22 -12.0 10173.157075411711
+-13 626923679 -15446 30 -13.0 8907.942987576691
-14 626923679 -13884 22 -14.0 10125.818731386042
--15 626923679 -16036 24 -15.0 9450.506254395024
+-15 626923679 -16036 24 -15.0 9450.506254395026
-16 626923679 -15154 21 -16.0 8884.207393686478
-17 626923679 -15922 19 -17.0 9944.104273894172
-18 626923679 -14863 24 -18.0 9638.430684071413
-19 626923679 -15935 25 -19.0 9967.22240685782
-2 626923679 -16277 20 -2.0 10800.090249507177
--20 626923679 -16126 24 -20.0 9868.92268080106
+-20 626923679 -16126 24 -20.0 9868.922680801063
-21 626923679 -16017 27 -21.0 9480.349236669877
-22 626923679 -14701 22 -22.0 8809.230165774987
-23 626923679 -16355 36 -23.345263230173213 9401.831290253447
-24 626923679 -16311 26 -24.0 9386.736402961187
--25 626923679 -15862 24 -25.0 9778.256724727018
--26 626923679 -15686 15 -26.0 10874.523900405318
+-25 626923679 -15862 24 -25.0 9778.25672472702
+-26 626923679 -15686 15 -26.0 10874.52390040532
-27 626923679 -14984 20 -27.0 8465.29660255097
-28 626923679 -15813 20 -28.0 9616.869413270924
--29 626923679 -14747 26 -29.0 9052.945656011721
+-29 626923679 -14747 26 -29.0 9052.945656011723
-3 626923679 -13632 16 -3.0 8836.215573422822
--30 626923679 -14863 23 -30.0 9193.941914019653
+-30 626923679 -14863 23 -30.0 9193.941914019651
-31 626923679 -15915 22 -31.0 9187.596784112568
--32 626923679 -15866 25 -32.0 9535.546396775915
+-32 626923679 -15866 25 -32.0 9535.546396775917
-33 626923679 -12779 21 -33.0 8854.331159704514
-34 626923679 -15450 29 -34.0 8708.243526705026
-35 626923679 -16059 23 -35.0 10136.580492864763
--36 626923679 -16208 23 -36.0 8773.547684436919
+-36 626923679 -16208 23 -36.0 8773.54768443692
-37 626923679 -14780 17 -37.0 10368.905538788269
-38 626923679 -14914 28 -38.0 8767.375358291503
-39 626923679 -15612 19 -39.0 9765.551806305297
@@ -1292,37 +1316,37 @@ POSTHOOK: Input: default@alltypes_orc
-45 626923679 -15027 21 -45.0 8567.489593562543
-46 626923679 -12427 21 -46.0 9182.943188188632
-47 626923679 -16096 19 -47.0 9011.009178780589
--48 626923679 -15462 26 -48.0 9913.883371354861
+-48 626923679 -15462 26 -48.0 9913.883371354863
-49 626923679 -14831 23 -49.0 9894.429191738676
-5 626923679 -15780 24 -5.0 10599.227726422314
-50 626923679 -14320 27 -50.0 8548.827748002343
-51 1073680599 -15734 1028 -51.0 9531.569305177045
-52 626923679 -16369 30 -52.0 8625.06871423408
--53 626923679 -15445 19 -53.0 9387.739325499799
--54 626923679 -14815 23 -54.0 9614.154026896626
+-53 626923679 -15445 19 -53.0 9387.7393254998
+-54 626923679 -14815 23 -54.0 9614.154026896624
-55 626923679 -13381 26 -55.0 9157.562103946742
--56 626923679 -11999 33 -56.0 9490.842152672341
--57 626923679 -14893 32 -57.0 8572.083461570477
--58 626923679 -15169 20 -58.0 9549.096672008198
--59 626923679 -15789 28 -59.0 9829.790704244733
+-56 626923679 -11999 33 -56.0 9490.84215267234
+-57 626923679 -14893 32 -57.0 8572.083461570479
+-58 626923679 -15169 20 -58.0 9549.096672008196
+-59 626923679 -15789 28 -59.0 9829.790704244735
-6 626923679 -15980 30 -6.0 10262.829252317424
-60 626923679 -15792 24 -60.0 9892.656196775464
--61 626923679 -15142 22 -61.0 9357.236187870849
--62 626923679 -15992 24 -62.0 9004.593091474135
+-61 626923679 -15142 22 -61.0 9357.23618787085
+-62 626923679 -15992 24 -62.0 9004.593091474137
-63 626923679 -12516 16 -63.0 9263.605837223322
-64 626923679 -15920 21 -64.0 9254.456539277186
--7 626923679 -14584 23 -7.0 9946.605446407746
--8 626923679 -14678 18 -8.0 9976.831992670684
+-7 626923679 -14584 23 -7.0 9946.605446407748
+-8 626923679 -14678 18 -8.0 9976.831992670686
-9 626923679 -15329 31 -9.0 8999.391457373968
0 626923679 -14254 24 0.0 10057.5018088718
-1 626923679 -14610 30 1.0 10016.486277900643
-10 626923679 -15887 26 10.0 9104.820520135108
+1 626923679 -14610 30 1.0 10016.486277900645
+10 626923679 -15887 26 10.0 9104.82052013511
11 1072654057 -14696 1035 11.0 9531.018991371746
12 626923679 -14642 18 12.0 9696.038286378725
13 626923679 -14771 26 13.0 8128.265919972384
14 626923679 -13367 28 14.0 9074.674998750581
-15 626923679 -16339 28 15.0 9770.473400901916
-16 626923679 -14001 26 16.0 10130.883606275334
+15 626923679 -16339 28 15.0 9770.473400901918
+16 626923679 -14001 26 16.0 10130.883606275338
17 626923679 -16109 22 16.73235294865627 1353416.3383574807
18 626923679 -15779 21 18.0 10820.004053788869
19 626923679 -16049 21 19.0 9423.560227007669
@@ -1333,28 +1357,28 @@ POSTHOOK: Input: default@alltypes_orc
23 626923679 -15514 24 23.0 8542.419116415425
24 626923679 -15086 24 24.0 9661.203790645088
25 626923679 -11349 23 25.0 8888.959012093468
-26 626923679 -14516 29 26.0 9123.125508880432
-27 626923679 -14965 24 27.0 9802.871860196345
+26 626923679 -14516 29 26.0 9123.125508880434
+27 626923679 -14965 24 27.0 9802.871860196343
28 626923679 -14455 20 28.0 9283.289383115296
29 626923679 -15892 16 29.0 9874.046501817154
-3 626923679 -16339 30 3.0 10483.526375885149
-30 626923679 -14111 27 30.0 10066.520234676527
+3 626923679 -16339 30 3.0 10483.526375885147
+30 626923679 -14111 27 30.0 10066.520234676529
31 626923679 -15960 24 31.0 10427.970184550613
32 626923679 -14044 24 32.0 8376.464579403413
-33 626923679 -14642 29 40.61776386607777 1304429.5939037625
-34 626923679 -15059 28 34.0 8756.731536033676
+33 626923679 -14642 29 40.61776386607777 1304429.593903763
+34 626923679 -15059 28 34.0 8756.731536033674
35 626923679 -16153 27 35.0 10351.008404963042
36 626923679 -15912 20 36.0 9475.257975138164
37 626923679 -12081 24 37.0 9017.860034890362
38 626923679 -15248 29 38.0 9900.256257785535
-39 626923679 -14887 28 39.0 10513.343644635232
-4 626923679 -15999 29 4.0 9516.189702058042
+39 626923679 -14887 28 39.0 10513.343644635233
+4 626923679 -15999 29 4.0 9516.189702058044
40 626923679 -15861 22 40.0 9283.318678549174
-41 626923679 -13480 21 41.0 9016.291129937847
+41 626923679 -13480 21 41.0 9016.291129937848
42 626923679 -15834 28 42.0 10318.01399719996
43 626923679 -15703 28 43.0 8757.796089055722
44 626923679 -11185 16 44.0 9425.076634933797
-45 626923679 -15228 18 45.0 9459.968668643689
+45 626923679 -15228 18 45.0 9459.968668643687
46 626923679 -15187 22 46.0 9685.908173160062
47 626923679 -16324 22 47.0 9822.220821743611
48 626923679 -16372 29 48.0 10079.286173063345
@@ -1365,18 +1389,18 @@ POSTHOOK: Input: default@alltypes_orc
52 626923679 -15450 20 52.0 9261.723648435052
53 626923679 -16217 30 53.0 9895.247408969733
54 626923679 -15245 16 54.0 9789.50878424882
-55 626923679 -15887 21 55.0 9826.38569192808
+55 626923679 -15887 21 55.0 9826.385691928082
56 626923679 -12631 21 56.0 8860.917133763547
57 626923679 -15620 25 57.0 9413.99393840875
58 626923679 -13627 20 58.0 9083.529665947459
-59 626923679 -16076 17 59.0 10117.44967077967
-6 626923679 -15948 30 6.0 9644.247255286113
+59 626923679 -16076 17 59.0 10117.449670779672
+6 626923679 -15948 30 6.0 9644.247255286115
60 626923679 -13606 23 60.0 8346.267436552042
-61 626923679 -15894 29 61.0 8785.714950987198
-62 626923679 -14307 17 62.0 9491.752726667326
+61 626923679 -15894 29 61.0 8785.7149509872
+62 626923679 -14307 17 62.0 9491.752726667324
7 626923679 -15839 25 7.0 10077.151640330823
8 1070764888 -15778 1034 8.0 9562.355155774725
-9 626923679 -13629 25 9.0 10157.217948808622
+9 626923679 -13629 25 9.0 10157.21794880862
NULL 1073418988 -16379 3115 NULL 305051.4870777435
PREHOOK: query: create table orcTbl (t1 tinyint, t2 tinyint)
stored as orc
http://git-wip-us.apache.org/repos/asf/hive/blob/5cb8867b/ql/src/test/results/clientpositive/spark/vectorization_not.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_not.q.out b/ql/src/test/results/clientpositive/spark/vectorization_not.q.out
index b5587ba..cf92a6f 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_not.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_not.q.out
@@ -55,4 +55,4 @@ WHERE (((cstring2 LIKE '%b%')
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--3.875652215945533E8 3.875652215945533E8 -3.875716535945533E8 1.436387455459401E9 3.875716535945533E8 0.0 2.06347151720204902E18 3.875716535945533E8 3.875652215945533E8 3.875716535945533E8 1.0 10934 -37224.52399241924 1.0517370547117279E9 -2.06347151720204902E18 1.5020929380914048E17 -64 64
+-3.875652215945533E8 3.875652215945533E8 -3.875716535945533E8 1.4363874554593627E9 3.875716535945533E8 0.0 2.0634715172019392E18 3.875716535945533E8 3.875652215945533E8 3.875716535945533E8 1.0 10934 -37224.52399241924 1.0516820315185745E9 -2.0634715172019392E18 1.5020929380914048E17 -64 64
http://git-wip-us.apache.org/repos/asf/hive/blob/5cb8867b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
index 6d52551..0cf6955 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
@@ -32,14 +32,14 @@ STAGE PLANS:
outputColumnNames: cbigint
Statistics: Num rows: 4096 Data size: 880654 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cbigint)
+ aggregations: sum(cbigint), count(cbigint)
mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -60,17 +60,21 @@ STAGE PLANS:
vectorized: true
Reduce Operator Tree:
Group By Operator
- aggregations: avg(VALUE._col0)
+ aggregations: sum(VALUE._col0), count(VALUE._col1)
mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: (_col0 / _col1) (type: double)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator