You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/03/31 04:45:36 UTC
[3/5] hive git commit: HIVE-19019: Vectorization: When vectorized,
orc_merge_incompat_schema.q throws HiveException "Not implemented
yet" from VectorExpressionWriterMap (Matt McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/fc48d721/ql/src/test/results/clientpositive/llap/vector_orc_null_check.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_null_check.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_null_check.q.out
new file mode 100644
index 0000000..4c2c4a0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_orc_null_check.q.out
@@ -0,0 +1,121 @@
+PREHOOK: query: create table listtable(l array<string>)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@listtable
+POSTHOOK: query: create table listtable(l array<string>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@listtable
+PREHOOK: query: create table listtable_orc(l array<string>) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@listtable_orc
+POSTHOOK: query: create table listtable_orc(l array<string>) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@listtable_orc
+PREHOOK: query: insert overwrite table listtable select array(null) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@listtable
+POSTHOOK: query: insert overwrite table listtable select array(null) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@listtable
+POSTHOOK: Lineage: listtable.l EXPRESSION []
+PREHOOK: query: insert overwrite table listtable_orc select * from listtable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@listtable
+PREHOOK: Output: default@listtable_orc
+POSTHOOK: query: insert overwrite table listtable_orc select * from listtable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@listtable
+POSTHOOK: Output: default@listtable_orc
+POSTHOOK: Lineage: listtable_orc.l SIMPLE [(listtable)listtable.FieldSchema(name:l, type:array<string>, comment:null), ]
+PREHOOK: query: explain vectorization expression
+select size(l) from listtable_orc limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select size(l) from listtable_orc limit 10
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: listtable_orc
+ Statistics: Num rows: 500 Data size: 913920 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: size(l) (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: VectorUDFAdaptor(size(l)) -> 2:int
+ Statistics: Num rows: 500 Data size: 913920 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 10 Data size: 18270 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 10 Data size: 18270 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select size(l) from listtable_orc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@listtable_orc
+#### A masked pattern was here ####
+POSTHOOK: query: select size(l) from listtable_orc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@listtable_orc
+#### A masked pattern was here ####
+-1
+-1
+-1
+-1
+-1
+-1
+-1
+-1
+-1
+-1
http://git-wip-us.apache.org/repos/asf/hive/blob/fc48d721/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out
index c299796..7be4d7d 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out
@@ -1,7 +1,9 @@
WARNING: Comparing a bigint and a double may result in a loss of precision.
-PREHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 7
+PREHOOK: query: explain vectorization
+SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 order by cbigint, cdouble limit 7
PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 7
+POSTHOOK: query: explain vectorization
+SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 order by cbigint, cdouble limit 7
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -15,6 +17,9 @@ STAGE PLANS:
Stage: Stage-1
Tez
#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
@@ -28,16 +33,11 @@ STAGE PLANS:
expressions: cbigint (type: bigint), cdouble (type: double)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1365 Data size: 16320 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 7
- Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint), _col1 (type: double)
+ sort order: ++
+ Statistics: Num rows: 1365 Data size: 16320 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -46,9 +46,32 @@ STAGE PLANS:
inputFormatFeatureSupport: []
featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1365 Data size: 16320 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 7
+ Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -57,26 +80,26 @@ STAGE PLANS:
ListSink
WARNING: Comparing a bigint and a double may result in a loss of precision.
-PREHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 7
+PREHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 order by cbigint, cdouble limit 7
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 limit 7
+POSTHOOK: query: SELECT cbigint, cdouble FROM alltypesorc WHERE cbigint < cdouble and cint > 0 order by cbigint, cdouble limit 7
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--1887561756 -10011.0
--1887561756 -13877.0
--1887561756 -2281.0
--1887561756 -8881.0
--1887561756 10361.0
--1887561756 1839.0
--1887561756 9531.0
+-1887561756 -15891.0
+-1887561756 -15951.0
+-1887561756 -16008.0
+-1887561756 -16183.0
+-1887561756 -16225.0
+-1887561756 -16243.0
+-1887561756 -16296.0
PREHOOK: query: explain vectorization detail
-select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 20
+select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble,csmallint limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 20
+select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble,csmallint limit 20
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -118,17 +141,16 @@ STAGE PLANS:
projectedOutputColumnNums: [0, 5, 1]
Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
- key expressions: _col0 (type: tinyint), _col1 (type: double)
- sort order: ++
+ key expressions: _col0 (type: tinyint), _col1 (type: double), _col2 (type: smallint)
+ sort order: +++
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
- keyColumnNums: [0, 5]
+ keyColumnNums: [0, 5, 1]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [1]
+ valueColumnNums: []
Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.3
- value expressions: _col2 (type: smallint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -151,19 +173,19 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- reduceColumnNullOrder: aa
- reduceColumnSortOrder: ++
+ reduceColumnNullOrder: aaa
+ reduceColumnSortOrder: +++
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
rowBatchContext:
dataColumnCount: 3
- dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:double, VALUE._col0:smallint
+ dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:double, KEY.reducesinkkey2:smallint
partitionColumnCount: 0
scratchColumnTypeNames: []
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), VALUE._col0 (type: smallint)
+ expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: smallint)
outputColumnNames: _col0, _col1, _col2
Select Vectorization:
className: VectorSelectOperator
@@ -193,11 +215,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 20
+PREHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble,csmallint limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble limit 20
+POSTHOOK: query: select ctinyint,cdouble,csmallint from alltypesorc where ctinyint is not null order by ctinyint,cdouble,csmallint limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -222,10 +244,10 @@ POSTHOOK: Input: default@alltypesorc
-64 -8080.0 -8080
-64 -9842.0 -9842
PREHOOK: query: explain vectorization detail
-select ctinyint,avg(cdouble + 1) from alltypesorc group by ctinyint order by ctinyint limit 20
+select ctinyint,avg(cdouble + 1) as cavg from alltypesorc group by ctinyint order by ctinyint, cavg limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select ctinyint,avg(cdouble + 1) from alltypesorc group by ctinyint order by ctinyint limit 20
+select ctinyint,avg(cdouble + 1) as cavg from alltypesorc group by ctinyint order by ctinyint, cavg limit 20
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -241,6 +263,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -279,14 +302,12 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
+ className: VectorReduceSinkLongOperator
keyColumnNums: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumnNums: [0]
valueColumnNums: [1, 2]
Statistics: Num rows: 128 Data size: 2436 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: double), _col2 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -344,22 +365,57 @@ STAGE PLANS:
projectedOutputColumnNums: [0, 3]
selectExpressions: DoubleColDivideLongColumn(col 1:double, col 2:bigint) -> 3:double
Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 20
- Limit Vectorization:
- className: VectorLimitOperator
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint), _col1 (type: double)
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumnNums: [0, 3]
native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: []
+ Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.3
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:double
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 20
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -367,11 +423,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint,avg(cdouble + 1) from alltypesorc group by ctinyint order by ctinyint limit 20
+PREHOOK: query: select ctinyint,avg(cdouble + 1) as cavg from alltypesorc group by ctinyint order by ctinyint, cavg limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint,avg(cdouble + 1) from alltypesorc group by ctinyint order by ctinyint limit 20
+POSTHOOK: query: select ctinyint,avg(cdouble + 1) as cavg from alltypesorc group by ctinyint order by ctinyint, cavg limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -396,10 +452,10 @@ POSTHOOK: Input: default@alltypesorc
-64 373.52941176470586
NULL 9370.0945309795
PREHOOK: query: explain vectorization detail
-select distinct(ctinyint) from alltypesorc limit 20
+select distinct(ctinyint) as cdistinct from alltypesorc order by cdistinct limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select distinct(ctinyint) from alltypesorc limit 20
+select distinct(ctinyint) as cdistinct from alltypesorc order by cdistinct limit 20
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -450,10 +506,11 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
Reduce Sink Vectorization:
- className: VectorReduceSinkLongOperator
+ className: VectorReduceSinkObjectHashOperator
keyColumnNums: [0]
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumnNums: [0]
valueColumnNums: []
Statistics: Num rows: 128 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.3
@@ -525,11 +582,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select distinct(ctinyint) from alltypesorc limit 20
+PREHOOK: query: select distinct(ctinyint) as cdistinct from alltypesorc order by cdistinct limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select distinct(ctinyint) from alltypesorc limit 20
+POSTHOOK: query: select distinct(ctinyint) as cdistinct from alltypesorc order by cdistinct limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -554,10 +611,10 @@ POSTHOOK: Input: default@alltypesorc
-64
NULL
PREHOOK: query: explain vectorization detail
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) as count_distinct from alltypesorc group by ctinyint order by ctinyint, count_distinct limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
+select ctinyint, count(distinct(cdouble)) as count_distinct from alltypesorc group by ctinyint order by ctinyint, count_distinct limit 20
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -573,6 +630,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -674,22 +732,57 @@ STAGE PLANS:
mode: complete
outputColumnNames: _col0, _col1
Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 20
- Limit Vectorization:
- className: VectorLimitOperator
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint), _col1 (type: bigint)
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumnNums: [0, 1]
native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: []
+ Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.3
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ reduceColumnNullOrder: aa
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:bigint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ Statistics: Num rows: 128 Data size: 1412 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 20
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
- Statistics: Num rows: 20 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -697,11 +790,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
+PREHOOK: query: select ctinyint, count(distinct(cdouble)) as count_distinct from alltypesorc group by ctinyint order by ctinyint, count_distinct limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20
+POSTHOOK: query: select ctinyint, count(distinct(cdouble)) as count_distinct from alltypesorc group by ctinyint order by ctinyint, count_distinct limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
@@ -726,10 +819,10 @@ POSTHOOK: Input: default@alltypesorc
-64 24
NULL 2932
PREHOOK: query: explain vectorization detail
-select ctinyint,cdouble from alltypesorc order by ctinyint limit 0
+select ctinyint,cdouble from alltypesorc order by ctinyint,cdouble limit 0
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select ctinyint,cdouble from alltypesorc order by ctinyint limit 0
+select ctinyint,cdouble from alltypesorc order by ctinyint,cdouble limit 0
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -745,19 +838,19 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select ctinyint,cdouble from alltypesorc order by ctinyint limit 0
+PREHOOK: query: select ctinyint,cdouble from alltypesorc order by ctinyint,cdouble limit 0
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select ctinyint,cdouble from alltypesorc order by ctinyint limit 0
+POSTHOOK: query: select ctinyint,cdouble from alltypesorc order by ctinyint,cdouble limit 0
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
PREHOOK: query: explain vectorization detail
-select cdouble, sum(ctinyint) as sum from alltypesorc where ctinyint is not null group by cdouble order by sum, cdouble limit 20
+select cdouble, sum(ctinyint) as csum from alltypesorc where ctinyint is not null group by cdouble order by csum, cdouble limit 20
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization detail
-select cdouble, sum(ctinyint) as sum from alltypesorc where ctinyint is not null group by cdouble order by sum, cdouble limit 20
+select cdouble, sum(ctinyint) as csum from alltypesorc where ctinyint is not null group by cdouble order by csum, cdouble limit 20
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -922,11 +1015,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select cdouble, sum(ctinyint) as sum from alltypesorc where ctinyint is not null group by cdouble order by sum, cdouble limit 20
+PREHOOK: query: select cdouble, sum(ctinyint) as csum from alltypesorc where ctinyint is not null group by cdouble order by csum, cdouble limit 20
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select cdouble, sum(ctinyint) as sum from alltypesorc where ctinyint is not null group by cdouble order by sum, cdouble limit 20
+POSTHOOK: query: select cdouble, sum(ctinyint) as csum from alltypesorc where ctinyint is not null group by cdouble order by csum, cdouble limit 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####