You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/01/25 10:41:44 UTC
hive git commit: HIVE-18521: Vectorization: query failing in reducer
VectorUDAFAvgDecimalPartial2 java.lang.ClassCastException StructTypeInfo -->
DecimalTypeInfo (Matt McCline, reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master cb866e894 -> 3f7127f59
HIVE-18521: Vectorization: query failing in reducer VectorUDAFAvgDecimalPartial2 java.lang.ClassCastException StructTypeInfo --> DecimalTypeInfo (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f7127f5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f7127f5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f7127f5
Branch: refs/heads/master
Commit: 3f7127f59b462667da42204133fbcd8ce22e743e
Parents: cb866e8
Author: Matt McCline <mm...@hortonworks.com>
Authored: Thu Jan 25 04:39:42 2018 -0600
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Thu Jan 25 04:41:04 2018 -0600
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../UDAFTemplates/VectorUDAFAvgDecimalMerge.txt | 6 +-
.../vector_groupby_grouping_sets3_dec.q | 40 ++
.../vector_groupby_grouping_sets3_dec.q.out | 545 +++++++++++++++++++
4 files changed, 590 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3f7127f5/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 1017249..1362079 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -687,6 +687,7 @@ minillaplocal.query.files=\
vector_groupby_grouping_sets1.q,\
vector_groupby_grouping_sets2.q,\
vector_groupby_grouping_sets3.q,\
+ vector_groupby_grouping_sets3_dec.q,\
vector_groupby_grouping_sets4.q,\
vector_groupby_grouping_sets5.q,\
vector_groupby_grouping_sets6.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/3f7127f5/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
index 5d3e422..8ab393c 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
@@ -97,7 +97,7 @@ public class <ClassName> extends VectorAggregateExpression {
#IF FINAL
transient private HiveDecimalWritable tempDecWritable;
-#ENDIF FINAL
+
DecimalTypeInfo outputDecimalTypeInfo;
@@ -110,6 +110,7 @@ public class <ClassName> extends VectorAggregateExpression {
* The precision of the SUM in the partial output
*/
private int sumPrecision;
+#ENDIF FINAL
// This constructor is used to momentarily create the object so match can be called.
public <ClassName>() {
@@ -128,10 +129,11 @@ public class <ClassName> extends VectorAggregateExpression {
}
private void init() {
+#IF FINAL
outputDecimalTypeInfo = (DecimalTypeInfo) outputTypeInfo;
sumScale = outputDecimalTypeInfo.scale();
sumPrecision = outputDecimalTypeInfo.precision();
-#IF FINAL
+
tempDecWritable = new HiveDecimalWritable();
#ENDIF FINAL
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3f7127f5/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets3_dec.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets3_dec.q b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets3_dec.q
new file mode 100644
index 0000000..1dff14c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_groupby_grouping_sets3_dec.q
@@ -0,0 +1,40 @@
+set hive.explain.user=false;
+SET hive.vectorized.execution.enabled=true;
+SET hive.vectorized.execution.reduce.enabled=true;
+set hive.fetch.task.conversion=none;
+set hive.cli.print.header=true;
+
+-- SORT_QUERY_RESULTS
+
+-- In this test, 2 files are loaded into table T1. The data contains rows with the same value of a and b,
+-- with different number of rows for a and b in each file. Since bucketizedHiveInputFormat is used,
+-- this tests that the aggregate function stores the partial aggregate state correctly even if an
+-- additional MR job is created for processing the grouping sets.
+CREATE TABLE T1_text(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/grouping_sets1.txt' INTO TABLE T1_text;
+LOAD DATA LOCAL INPATH '../../data/files/grouping_sets2.txt' INTO TABLE T1_text;
+
+CREATE TABLE T1 STORED AS ORC AS SELECT a, b, cast(c as decimal(10,2)) as c_dec FROM T1_text;
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set hive.new.job.grouping.set.cardinality = 30;
+
+-- The query below will execute in a single MR job, since 4 rows are generated per input row
+-- (cube of a,b will lead to (a,b), (a, null), (null, b) and (null, null) and
+-- hive.new.job.grouping.set.cardinality is more than 4.
+EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube;
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by cube(a, b);
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube;
+
+set hive.new.job.grouping.set.cardinality=2;
+
+-- The query below will execute in 2 MR jobs, since hive.new.job.grouping.set.cardinality is set to 2.
+-- The partial aggregation state should be maintained correctly across MR jobs.
+EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube;
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/3f7127f5/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets3_dec.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets3_dec.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets3_dec.q.out
new file mode 100644
index 0000000..d98ce9b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets3_dec.q.out
@@ -0,0 +1,545 @@
+PREHOOK: query: CREATE TABLE T1_text(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T1_text
+POSTHOOK: query: CREATE TABLE T1_text(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T1_text
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/grouping_sets1.txt' INTO TABLE T1_text
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t1_text
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/grouping_sets1.txt' INTO TABLE T1_text
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t1_text
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/grouping_sets2.txt' INTO TABLE T1_text
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t1_text
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/grouping_sets2.txt' INTO TABLE T1_text
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t1_text
+PREHOOK: query: CREATE TABLE T1 STORED AS ORC AS SELECT a, b, cast(c as decimal(10,2)) as c_dec FROM T1_text
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t1_text
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T1
+POSTHOOK: query: CREATE TABLE T1 STORED AS ORC AS SELECT a, b, cast(c as decimal(10,2)) as c_dec FROM T1_text
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t1_text
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T1
+POSTHOOK: Lineage: t1.a SIMPLE [(t1_text)t1_text.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: t1.b SIMPLE [(t1_text)t1_text.FieldSchema(name:b, type:string, comment:null), ]
+POSTHOOK: Lineage: t1.c_dec EXPRESSION [(t1_text)t1_text.FieldSchema(name:c, type:string, comment:null), ]
+a b c_dec
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:string, 1:b:string, 2:c_dec:decimal(10,2), 3:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
+ Select Operator
+ expressions: a (type: string), b (type: string), c_dec (type: decimal(10,2))
+ outputColumnNames: a, b, c_dec
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: avg(c_dec), count()
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimal(col 2:decimal(10,2)) -> struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string, ConstantVectorExpression(val 0) -> 4:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0, 1]
+ keys: a (type: string), b (type: string), 0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: [3, 4]
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>), _col4 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: a:string, b:string, c_dec:decimal(10,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aaa
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY._col0:string, KEY._col1:string, KEY._col2:int, VALUE._col0:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VALUE._col1:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: avg(VALUE._col0), count(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimalFinal(col 3:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>) -> decimal(14,6), VectorUDAFCountMerge(col 4:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:string, col 2:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0, 1]
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ pruneGroupingSetId: true
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(14,6)), _col4 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3]
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by cube(a, b)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by cube(a, b)
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:string, 1:b:string, 2:c_dec:decimal(10,2), 3:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
+ Select Operator
+ expressions: a (type: string), b (type: string), c_dec (type: decimal(10,2))
+ outputColumnNames: a, b, c_dec
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: avg(c_dec), count()
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimal(col 2:decimal(10,2)) -> struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string, ConstantVectorExpression(val 0) -> 4:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0, 1]
+ keys: a (type: string), b (type: string), 0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: [3, 4]
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>), _col4 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: a:string, b:string, c_dec:decimal(10,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aaa
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY._col0:string, KEY._col1:string, KEY._col2:int, VALUE._col0:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VALUE._col1:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: avg(VALUE._col0), count(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimalFinal(col 3:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>) -> decimal(14,6), VectorUDAFCountMerge(col 4:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:string, col 2:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0, 1]
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ pruneGroupingSetId: true
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(14,6)), _col4 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3]
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+a b _c2 _c3
+1 1 3.000000 2
+1 2 2.000000 1
+1 NULL 2.666667 3
+2 2 5.333333 3
+2 3 5.000000 2
+2 NULL 5.200000 5
+3 2 8.000000 1
+3 NULL 8.000000 1
+5 1 2.000000 1
+5 NULL 2.000000 1
+8 1 1.000000 2
+8 NULL 1.000000 2
+NULL 1 2.000000 5
+NULL 2 5.200000 5
+NULL 3 5.000000 2
+NULL NULL 3.833333 12
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+POSTHOOK: type: QUERY
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:string, 1:b:string, 2:c_dec:decimal(10,2), 3:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
+ Select Operator
+ expressions: a (type: string), b (type: string), c_dec (type: decimal(10,2))
+ outputColumnNames: a, b, c_dec
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: avg(c_dec), count()
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimal(col 2:decimal(10,2)) -> struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0, 1]
+ keys: a (type: string), b (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumnNums: [0, 1]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: [2, 3]
+ Statistics: Num rows: 12 Data size: 5760 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>), _col3 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: a:string, b:string, c_dec:decimal(10,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY._col0:string, KEY._col1:string, VALUE._col0:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VALUE._col1:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint]
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: avg(VALUE._col0), count(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimalPartial2(col 2:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>) -> struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VectorUDAFCountMerge(col 3:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: PARTIALS
+ keyExpressions: col 0:string, col 1:string, ConstantVectorExpression(val 0) -> 4:int
+ native: false
+ vectorProcessingMode: STREAMING
+ projectedOutputColumnNums: [0, 1]
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), 0 (type: int)
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: [3, 4]
+ Statistics: Num rows: 48 Data size: 23040 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>), _col4 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aaa
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY._col0:string, KEY._col1:string, KEY._col2:int, VALUE._col0:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>, VALUE._col1:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: avg(VALUE._col0), count(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFAvgDecimalFinal(col 3:struct<count:bigint,sum:decimal(20,2),input:decimal(10,2)>) -> decimal(14,6), VectorUDAFCountMerge(col 4:bigint) -> bigint
+ className: VectorGroupByOperator
+ groupByMode: FINAL
+ keyExpressions: col 0:string, col 1:string, col 2:int
+ native: false
+ vectorProcessingMode: STREAMING
+ projectedOutputColumnNums: [0, 1]
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int)
+ mode: final
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ pruneGroupingSetId: true
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(14,6)), _col4 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3]
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 24 Data size: 11520 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a, b, avg(c_dec), count(*) from T1 group by a, b with cube
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+a b _c2 _c3
+1 1 3.000000 2
+1 2 2.000000 1
+1 NULL 2.666667 3
+2 2 5.333333 3
+2 3 5.000000 2
+2 NULL 5.200000 5
+3 2 8.000000 1
+3 NULL 8.000000 1
+5 1 2.000000 1
+5 NULL 2.000000 1
+8 1 1.000000 2
+8 NULL 1.000000 2
+NULL 1 2.000000 5
+NULL 2 5.200000 5
+NULL 3 5.000000 2
+NULL NULL 3.833333 12