You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/02/16 15:52:18 UTC
[04/32] hive git commit: HIVE-18622: Vectorization: IF Statements,
Comparisons, and more do not handle NULLs correctly (Matt McCline,
reviewed by Sergey Shelukhin, Deepak Jaiswal, Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out b/ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out
deleted file mode 100644
index fe68e5c..0000000
--- a/ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out
+++ /dev/null
@@ -1,2376 +0,0 @@
-PREHOOK: query: DROP TABLE IF EXISTS e011_01
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS e011_01
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: DROP TABLE IF EXISTS e011_02
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS e011_02
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: DROP TABLE IF EXISTS e011_03
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS e011_03
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE e011_01 (
- c1 decimal(15,2),
- c2 decimal(15,2))
- STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_01
-POSTHOOK: query: CREATE TABLE e011_01 (
- c1 decimal(15,2),
- c2 decimal(15,2))
- STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_01
-PREHOOK: query: CREATE TABLE e011_02 (
- c1 decimal(15,2),
- c2 decimal(15,2))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_02
-POSTHOOK: query: CREATE TABLE e011_02 (
- c1 decimal(15,2),
- c2 decimal(15,2))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_02
-PREHOOK: query: CREATE TABLE e011_03 (
- c1 decimal(15,2),
- c2 decimal(15,2))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_03
-POSTHOOK: query: CREATE TABLE e011_03 (
- c1 decimal(15,2),
- c2 decimal(15,2))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_03
-PREHOOK: query: CREATE TABLE e011_01_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
- STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_01_small
-POSTHOOK: query: CREATE TABLE e011_01_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
- STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_01_small
-PREHOOK: query: CREATE TABLE e011_02_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_02_small
-POSTHOOK: query: CREATE TABLE e011_02_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_02_small
-PREHOOK: query: CREATE TABLE e011_03_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@e011_03_small
-POSTHOOK: query: CREATE TABLE e011_03_small (
- c1 decimal(7,2),
- c2 decimal(7,2))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@e011_03_small
-PREHOOK: query: LOAD DATA
- LOCAL INPATH '../../data/files/e011_01.txt'
- OVERWRITE
- INTO TABLE e011_01
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@e011_01
-POSTHOOK: query: LOAD DATA
- LOCAL INPATH '../../data/files/e011_01.txt'
- OVERWRITE
- INTO TABLE e011_01
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@e011_01
-PREHOOK: query: INSERT INTO TABLE e011_02
- SELECT c1, c2
- FROM e011_01
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Output: default@e011_02
-POSTHOOK: query: INSERT INTO TABLE e011_02
- SELECT c1, c2
- FROM e011_01
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Output: default@e011_02
-POSTHOOK: Lineage: e011_02.c1 SIMPLE [(e011_01)e011_01.FieldSchema(name:c1, type:decimal(15,2), comment:null), ]
-POSTHOOK: Lineage: e011_02.c2 SIMPLE [(e011_01)e011_01.FieldSchema(name:c2, type:decimal(15,2), comment:null), ]
-c1 c2
-PREHOOK: query: INSERT INTO TABLE e011_03
- SELECT c1, c2
- FROM e011_01
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Output: default@e011_03
-POSTHOOK: query: INSERT INTO TABLE e011_03
- SELECT c1, c2
- FROM e011_01
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Output: default@e011_03
-POSTHOOK: Lineage: e011_03.c1 SIMPLE [(e011_01)e011_01.FieldSchema(name:c1, type:decimal(15,2), comment:null), ]
-POSTHOOK: Lineage: e011_03.c2 SIMPLE [(e011_01)e011_01.FieldSchema(name:c2, type:decimal(15,2), comment:null), ]
-c1 c2
-PREHOOK: query: LOAD DATA
- LOCAL INPATH '../../data/files/e011_01.txt'
- OVERWRITE
- INTO TABLE e011_01_small
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@e011_01_small
-POSTHOOK: query: LOAD DATA
- LOCAL INPATH '../../data/files/e011_01.txt'
- OVERWRITE
- INTO TABLE e011_01_small
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@e011_01_small
-PREHOOK: query: INSERT INTO TABLE e011_02_small
- SELECT c1, c2
- FROM e011_01_small
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-PREHOOK: Output: default@e011_02_small
-POSTHOOK: query: INSERT INTO TABLE e011_02_small
- SELECT c1, c2
- FROM e011_01_small
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-POSTHOOK: Output: default@e011_02_small
-POSTHOOK: Lineage: e011_02_small.c1 SIMPLE [(e011_01_small)e011_01_small.FieldSchema(name:c1, type:decimal(7,2), comment:null), ]
-POSTHOOK: Lineage: e011_02_small.c2 SIMPLE [(e011_01_small)e011_01_small.FieldSchema(name:c2, type:decimal(7,2), comment:null), ]
-c1 c2
-PREHOOK: query: INSERT INTO TABLE e011_03_small
- SELECT c1, c2
- FROM e011_01_small
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-PREHOOK: Output: default@e011_03_small
-POSTHOOK: query: INSERT INTO TABLE e011_03_small
- SELECT c1, c2
- FROM e011_01_small
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-POSTHOOK: Output: default@e011_03_small
-POSTHOOK: Lineage: e011_03_small.c1 SIMPLE [(e011_01_small)e011_01_small.FieldSchema(name:c1, type:decimal(7,2), comment:null), ]
-POSTHOOK: Lineage: e011_03_small.c2 SIMPLE [(e011_01_small)e011_01_small.FieldSchema(name:c2, type:decimal(7,2), comment:null), ]
-c1 c2
-PREHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Output: default@e011_01
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Output: default@e011_01
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_02
-PREHOOK: Output: default@e011_02
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_02
-POSTHOOK: Output: default@e011_02
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_03
-PREHOOK: Output: default@e011_03
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_03
-POSTHOOK: Output: default@e011_03
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: ANALYZE TABLE e011_01_small COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-PREHOOK: Output: default@e011_01_small
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_01_small COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-POSTHOOK: Output: default@e011_01_small
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: ANALYZE TABLE e011_02_small COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_02_small
-PREHOOK: Output: default@e011_02_small
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_02_small COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_02_small
-POSTHOOK: Output: default@e011_02_small
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: ANALYZE TABLE e011_03_small COMPUTE STATISTICS FOR COLUMNS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_03_small
-PREHOOK: Output: default@e011_03_small
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE e011_03_small COMPUTE STATISTICS FOR COLUMNS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_03_small
-POSTHOOK: Output: default@e011_03_small
-#### A masked pattern was here ####
-_c0 _c1
-PREHOOK: query: explain vectorization detail
-select sum(sum(c1)) over() from e011_01
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(c1)) over() from e011_01
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:c1:decimal(15,2)/DECIMAL_64, 1:c2:decimal(15,2)/DECIMAL_64, 2:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
- Select Operator
- expressions: c1 (type: decimal(15,2))
- outputColumnNames: c1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0]
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(c1)
- Group By Vectorization:
- aggregators: VectorUDAFSumDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> decimal(25,2)
- className: VectorGroupByOperator
- groupByMode: HASH
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: [DECIMAL_64]
- featureSupportInUse: [DECIMAL_64]
- inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 2
- includeColumns: [0]
- dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(25,2)]
- Reduce Output Operator
- key expressions: 0 (type: int)
- sort order: +
- Map-reduce partition columns: 0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 1
- includeColumns: [0]
- dataColumns: _col0:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, bigint]
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: decimal(25,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(25,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: 0 ASC NULLS FIRST
- partition by: 0
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col0
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(35,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(c1)) over() from e011_01
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(c1)) over() from e011_01
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-#### A masked pattern was here ####
-_c0
-16.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01
- group by e011_01.c1, e011_01.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01
- group by e011_01.c1, e011_01.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:c1:decimal(15,2)/DECIMAL_64, 1:c2:decimal(15,2)/DECIMAL_64, 2:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
- Select Operator
- expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- outputColumnNames: c1, c2
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(c1)
- Group By Vectorization:
- aggregators: VectorUDAFSumDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> decimal(25,2)
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(15,2)/DECIMAL_64) -> 3:decimal(15,2), ConvertDecimal64ToDecimal(col 1:decimal(15,2)/DECIMAL_64) -> 4:decimal(15,2)
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
- keys: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: [DECIMAL_64]
- featureSupportInUse: [DECIMAL_64]
- inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 2
- includeColumns: [0, 1]
- dataColumns: c1:decimal(15,2)/DECIMAL_64, c2:decimal(15,2)/DECIMAL_64
- partitionColumnCount: 0
- scratchColumnTypeNames: [decimal(15,2), decimal(15,2)]
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:decimal(25,2)]
- Reduce Output Operator
- key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: decimal(25,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(35,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01
- group by e011_01.c1, e011_01.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01
- group by e011_01.c1, e011_01.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-#### A masked pattern was here ####
-_c0
-1.00
-3.00
-5.00
-7.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(e011_01.c1)) over(
- partition by e011_01.c2 order by e011_01.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_01.c1, e011_01.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(e011_01.c1)) over(
- partition by e011_01.c2 order by e011_01.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_01.c1, e011_01.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-3 depends on stages: Stage-2
- Stage-0 depends on stages: Stage-3
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(15,2))
- TableScan
- alias: e011_03
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2))
- outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Map Vectorization:
- enabled: false
- enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: decimal(15,2))
- 1 _col0 (type: decimal(15,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col0)
- keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:decimal(25,2)]
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:decimal(25,2)]
- Reduce Output Operator
- key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: decimal(25,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(35,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(e011_01.c1)) over(
- partition by e011_01.c2 order by e011_01.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_01.c1, e011_01.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(e011_01.c1)) over(
- partition by e011_01.c2 order by e011_01.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_01.c1, e011_01.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-_c0
-1.00
-3.00
-5.00
-7.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(e011_01.c1)) over(
- partition by e011_03.c2 order by e011_03.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c1, e011_03.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(e011_01.c1)) over(
- partition by e011_03.c2 order by e011_03.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c1, e011_03.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-3 depends on stages: Stage-2
- Stage-0 depends on stages: Stage-3
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_03
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(15,2))
- TableScan
- alias: e011_01
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2))
- outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Map Vectorization:
- enabled: false
- enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: decimal(15,2))
- 1 _col0 (type: decimal(15,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col2)
- keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:decimal(25,2)]
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:decimal(25,2)]
- Reduce Output Operator
- key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col1 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(25,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:decimal(25,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: decimal(25,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(35,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(e011_01.c1)) over(
- partition by e011_03.c2 order by e011_03.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c1, e011_03.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(e011_01.c1)) over(
- partition by e011_03.c2 order by e011_03.c1)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c1, e011_03.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-_c0
-1.00
-3.00
-5.00
-7.00
-PREHOOK: query: explain vectorization detail
-select sum(corr(e011_01.c1, e011_03.c1))
- over(partition by e011_01.c2 order by e011_03.c2)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c2, e011_01.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(corr(e011_01.c1, e011_03.c1))
- over(partition by e011_01.c2 order by e011_03.c2)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c2, e011_01.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(15,2))
- TableScan
- alias: e011_03
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(15,2))
- Map Vectorization:
- enabled: false
- enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: decimal(15,2))
- 1 _col0 (type: decimal(15,2))
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: corr(_col0, _col2)
- keys: _col1 (type: decimal(15,2)), _col3 (type: decimal(15,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(15,2), 1:_col1:decimal(15,2), 2:_col2:struct<count:bigint,xavg:double,yavg:double,xvar:double,yvar:double,covar:double>]
- Reduce Output Operator
- key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(15,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: struct<count:bigint,xavg:double,yavg:double,xvar:double,yvar:double,covar:double>)
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(15,2), _col1:decimal(15,2), _col2:struct<count:bigint,xavg:double,yavg:double,xvar:double,yvar:double,covar:double>
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: corr(VALUE._col0)
- keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)), _col2 (type: double)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(15,2), _col1: decimal(15,2), _col2: double
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumDouble
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: double)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(corr(e011_01.c1, e011_03.c1))
- over(partition by e011_01.c2 order by e011_03.c2)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c2, e011_01.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01
-PREHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(corr(e011_01.c1, e011_03.c1))
- over(partition by e011_01.c2 order by e011_03.c2)
- from e011_01
- join e011_03 on e011_01.c1 = e011_03.c1
- group by e011_03.c2, e011_01.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01
-POSTHOOK: Input: default@e011_03
-#### A masked pattern was here ####
-sum_window_0
-NULL
-NULL
-NULL
-NULL
-PREHOOK: query: explain vectorization detail
-select sum(sum(c1)) over() from e011_01_small
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(c1)) over() from e011_01_small
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01_small
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:c1:decimal(7,2)/DECIMAL_64, 1:c2:decimal(7,2)/DECIMAL_64, 2:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
- Select Operator
- expressions: c1 (type: decimal(7,2))
- outputColumnNames: c1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0]
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(c1)
- Group By Vectorization:
- aggregators: VectorUDAFSumDecimal64(col 0:decimal(7,2)/DECIMAL_64) -> decimal(17,2)/DECIMAL_64
- className: VectorGroupByOperator
- groupByMode: HASH
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: [DECIMAL_64]
- featureSupportInUse: [DECIMAL_64]
- inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 2
- includeColumns: [0]
- dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(17,2)]
- Reduce Output Operator
- key expressions: 0 (type: int)
- sort order: +
- Map-reduce partition columns: 0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 1
- includeColumns: [0]
- dataColumns: _col0:decimal(17,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, bigint]
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: decimal(17,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(17,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: 0 ASC NULLS FIRST
- partition by: 0
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col0
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(27,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(c1)) over() from e011_01_small
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(c1)) over() from e011_01_small
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-#### A masked pattern was here ####
-_c0
-16.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01_small
- group by e011_01_small.c1, e011_01_small.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01_small
- group by e011_01_small.c1, e011_01_small.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01_small
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:c1:decimal(7,2)/DECIMAL_64, 1:c2:decimal(7,2)/DECIMAL_64, 2:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
- Select Operator
- expressions: c1 (type: decimal(7,2)), c2 (type: decimal(7,2))
- outputColumnNames: c1, c2
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(c1)
- Group By Vectorization:
- aggregators: VectorUDAFSumDecimal64(col 0:decimal(7,2)/DECIMAL_64) -> decimal(17,2)/DECIMAL_64
- className: VectorGroupByOperator
- groupByMode: HASH
- keyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(7,2)/DECIMAL_64) -> 3:decimal(7,2), ConvertDecimal64ToDecimal(col 1:decimal(7,2)/DECIMAL_64) -> 4:decimal(7,2)
- native: false
- vectorProcessingMode: HASH
- projectedOutputColumnNums: [0]
- keys: c1 (type: decimal(7,2)), c2 (type: decimal(7,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: [DECIMAL_64]
- featureSupportInUse: [DECIMAL_64]
- inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 2
- includeColumns: [0, 1]
- dataColumns: c1:decimal(7,2)/DECIMAL_64, c2:decimal(7,2)/DECIMAL_64
- partitionColumnCount: 0
- scratchColumnTypeNames: [decimal(7,2), decimal(7,2)]
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(7,2)), KEY._col1 (type: decimal(7,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(7,2), 1:_col1:decimal(7,2), 2:_col2:decimal(17,2)]
- Reduce Output Operator
- key expressions: _col1 (type: decimal(7,2)), _col0 (type: decimal(7,2))
- sort order: ++
- Map-reduce partition columns: _col1 (type: decimal(7,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(7,2), _col1:decimal(7,2), _col2:decimal(17,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey1 (type: decimal(7,2)), KEY.reducesinkkey0 (type: decimal(7,2)), VALUE._col0 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(7,2), _col1: decimal(7,2), _col2: decimal(17,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(27,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01_small
- group by e011_01_small.c1, e011_01_small.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(c1)) over(
- partition by c2 order by c1)
- from e011_01_small
- group by e011_01_small.c1, e011_01_small.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-#### A masked pattern was here ####
-_c0
-1.00
-3.00
-5.00
-7.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(e011_01_small.c1)) over(
- partition by e011_01_small.c2 order by e011_01_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_01_small.c1, e011_01_small.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(e011_01_small.c1)) over(
- partition by e011_01_small.c2 order by e011_01_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_01_small.c1, e011_01_small.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-3 depends on stages: Stage-2
- Stage-0 depends on stages: Stage-3
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: e011_01_small
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(7,2)), c2 (type: decimal(7,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(7,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(7,2))
- Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(7,2))
- TableScan
- alias: e011_03_small
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: c1 is not null (type: boolean)
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c1 (type: decimal(7,2))
- outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: decimal(7,2))
- sort order: +
- Map-reduce partition columns: _col0 (type: decimal(7,2))
- Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Map Vectorization:
- enabled: false
- enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: decimal(7,2))
- 1 _col0 (type: decimal(7,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col0)
- keys: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2))
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(7,2), 1:_col1:decimal(7,2), 2:_col2:decimal(17,2)]
- Reduce Output Operator
- key expressions: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2))
- sort order: ++
- Map-reduce partition columns: _col0 (type: decimal(7,2)), _col1 (type: decimal(7,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(7,2), _col1:decimal(7,2), _col2:decimal(17,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(7,2)), KEY._col1 (type: decimal(7,2))
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- TableScan Vectorization:
- native: true
- vectorizationSchemaColumns: [0:_col0:decimal(7,2), 1:_col1:decimal(7,2), 2:_col2:decimal(17,2)]
- Reduce Output Operator
- key expressions: _col1 (type: decimal(7,2)), _col0 (type: decimal(7,2))
- sort order: ++
- Map-reduce partition columns: _col1 (type: decimal(7,2))
- Reduce Sink Vectorization:
- className: VectorReduceSinkOperator
- native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Execution mode: vectorized
- Map Vectorization:
- enabled: true
- enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
- inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- includeColumns: [0, 1, 2]
- dataColumns: _col0:decimal(7,2), _col1:decimal(7,2), _col2:decimal(17,2)
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Vectorization:
- enabled: false
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
- enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey1 (type: decimal(7,2)), KEY.reducesinkkey0 (type: decimal(7,2)), VALUE._col0 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col0: decimal(7,2), _col1: decimal(7,2), _col2: decimal(17,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col0 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: sum_window_0
- arguments: _col2
- name: sum
- window function: GenericUDAFSumHiveDecimal
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sum_window_0 (type: decimal(27,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(sum(e011_01_small.c1)) over(
- partition by e011_01_small.c2 order by e011_01_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_01_small.c1, e011_01_small.c2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@e011_01_small
-PREHOOK: Input: default@e011_03_small
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(sum(e011_01_small.c1)) over(
- partition by e011_01_small.c2 order by e011_01_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_01_small.c1, e011_01_small.c2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@e011_01_small
-POSTHOOK: Input: default@e011_03_small
-#### A masked pattern was here ####
-_c0
-1.00
-3.00
-5.00
-7.00
-PREHOOK: query: explain vectorization detail
-select sum(sum(e011_01_small.c1)) over(
- partition by e011_03_small.c2 order by e011_03_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_03_small.c1, e011_03_small.c2
-PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization detail
-select sum(sum(e011_01_small.c1)) over(
- partition by e011_03_small.c2 order by e011_03_small.c1)
- from e011_01_small
- join e011_03_small on e011_01_small.c1 = e011_03_small.c1
- group by e011_03_small.c1, e011_03_small.c2
-POSTHOOK: type: QUERY
-Explain
-PLAN VECTORIZATION:
- enabled: true
- enabledConditionsMet: [hive.vectorized.execution.enabled
<TRUNCATED>