You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/03/31 16:01:42 UTC
[1/3] hive git commit: HIVE-19037: Vectorization: Miscellaneous
cleanup (Matt McCline, reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master fc48d7218 -> 9d0f9c07e
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
index 7dde2ec..2581311 100644
--- a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
@@ -221,7 +221,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Reducer 2
Execution mode: vectorized, llap
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
index d8ab3c5..b29fd4b 100644
--- a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out
@@ -151,7 +151,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Reducer 2
Execution mode: vectorized, llap
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out
index fc13ce1..81c6370 100644
--- a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out
@@ -32,10 +32,10 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@b
POSTHOOK: Lineage: b.s SCRIPT []
-PREHOOK: query: explain vectorization expression
+PREHOOK: query: explain vectorization detail
select NULL from a
PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization expression
+POSTHOOK: query: explain vectorization detail
select NULL from a
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
@@ -56,25 +56,45 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:s:string, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: ConstantVectorExpression(val null) -> 2:void
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 1
+ includeColumns: []
+ dataColumns: s:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [void]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_nvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_nvl.q.out b/ql/src/test/results/clientpositive/llap/vector_nvl.q.out
index 2bbbbe4..5a312fc 100644
--- a/ql/src/test/results/clientpositive/llap/vector_nvl.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_nvl.q.out
@@ -314,28 +314,44 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [13]
+ selectExpressions: ConstantVectorExpression(val null) -> 13:void
Statistics: Num rows: 12288 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out
index ebf6853..f1a4ea3 100644
--- a/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out
@@ -155,14 +155,18 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.orc_create_complex
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: no inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ vectorizationSupportRemovedReasons: [DECIMAL_64 disabled because LLAP is enabled]
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
- notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[strct].a
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
Stage: Stage-2
Dependency Collection
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out
index 7ff4bb1..0b645ab 100644
--- a/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out
@@ -733,28 +733,45 @@ STAGE PLANS:
alias: nested_tbl_1
Pruned Column Paths: s1.f2, s2.f8
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 8:string, val foo)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 1:int) -> 8:string), FilterLongColGreaterLongScalar(col 11:int, val 1)(children: VectorUDFAdaptor(size(s2.f8.f10))(children: VectorUDFStructField(col 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 1:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 10:array<int>) -> 11:int), SelectColumnIsTrue(col 11:boolean)(children: VectorUDFAdaptor(s2.f8.f11['key1'])(children: VectorUDFStructField(col 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 2:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:in
t) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 12:map<string,boolean>) -> 11:boolean))
predicate: ((s1.f2 = 'foo') and (size(s2.f8.f10) > 1) and s2.f8.f11['key1']) (type: boolean)
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s2.f8 (type: struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9]
+ selectExpressions: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f10
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
Stage: Stage-0
Fetch Operator
@@ -770,7 +787,7 @@ POSTHOOK: query: SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2.
POSTHOOK: type: QUERY
POSTHOOK: Input: default@nested_tbl_1
#### A masked pattern was here ####
-{"f9":true,"f10":[10,11],"f11":{"key2":false,"key1":true}}
+{"f9":true,"f10":[10,11],"f11":{"key1":true,"key2":false}}
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT col1, col2 FROM nested_tbl_1
LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1
@@ -957,25 +974,38 @@ STAGE PLANS:
alias: nested_tbl_1
Pruned Column Paths: s2.f8.f10, s1.f3.f4
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
Select Operator
expressions: (s2.f8.f10[1] pmod s1.f3.f4) (type: int)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [13]
+ selectExpressions: VectorUDFAdaptor((s2.f8.f10[1] pmod s1.f3.f4))(children: ListIndexColScalar(col 9:array<int>, col 1:int)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 1:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 9:array<int>) -> 10:int, VectorUDFStructField(col 11:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 11:struct<f4:int,f5:double>) -> 12:int) -> 13:int
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s1].f3.f4
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
Stage: Stage-0
Fetch Operator
@@ -1700,14 +1730,32 @@ STAGE PLANS:
alias: t2
Pruned Column Paths: s1.f6, s2.f8.f9
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsTrue(col 10:boolean)(children: VectorUDFAdaptor((s2.f8.f9 and s1.f6 is not null and s2.f8.f9 is not null))(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 9:boolean, IsNotNull(col 10:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 10:int) -> 11:boolean, IsNotNull(col 10:boolean)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 10:boolean) -> 12:boolean) ->
10:boolean)
predicate: (s1.f6 is not null and s2.f8.f9 and s2.f8.f9 is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f6 (type: int), (s2.f8.f9 = true) (type: boolean)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 12]
+ selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 9:int, LongColEqualLongScalar(col 11:boolean, val 1)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 11:boolean) -> 12:boolean
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
Group By Operator
+ Group By Vectorization:
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 9:int, col 12:boolean
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: []
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
@@ -1716,15 +1764,22 @@ STAGE PLANS:
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f9
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
Reducer 2
Execution mode: llap
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
index 93065d9..3bc9806 100644
--- a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out
@@ -148,7 +148,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Stage: Stage-2
@@ -656,7 +656,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Stage: Stage-2
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_case_when_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/vector_case_when_1.q.out
index f17c462..b2ac291 100644
--- a/ql/src/test/results/clientpositive/vector_case_when_1.q.out
+++ b/ql/src/test/results/clientpositive/vector_case_when_1.q.out
@@ -213,7 +213,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Reduce Vectorization:
enabled: false
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_case_when_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/vector_case_when_2.q.out
index 89c16b0..159c983 100644
--- a/ql/src/test/results/clientpositive/vector_case_when_2.q.out
+++ b/ql/src/test/results/clientpositive/vector_case_when_2.q.out
@@ -143,7 +143,7 @@ STAGE PLANS:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected primitive type category VOID
+ notVectorizedReason: SELECT operator: Unexpected hive type name void
vectorized: false
Reduce Vectorization:
enabled: false
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_null_projection.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_null_projection.q.out b/ql/src/test/results/clientpositive/vector_null_projection.q.out
index 8e491ff..f0f444f 100644
--- a/ql/src/test/results/clientpositive/vector_null_projection.q.out
+++ b/ql/src/test/results/clientpositive/vector_null_projection.q.out
@@ -32,10 +32,10 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@b
POSTHOOK: Lineage: b.s SCRIPT []
-PREHOOK: query: explain vectorization expression
+PREHOOK: query: explain vectorization detail
select NULL from a
PREHOOK: type: QUERY
-POSTHOOK: query: explain vectorization expression
+POSTHOOK: query: explain vectorization detail
select NULL from a
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
@@ -53,23 +53,44 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:s:string, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2]
+ selectExpressions: ConstantVectorExpression(val null) -> 2:void
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 1
+ includeColumns: []
+ dataColumns: s:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [void]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_nvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_nvl.q.out b/ql/src/test/results/clientpositive/vector_nvl.q.out
index 11eacc2..467dc7f 100644
--- a/ql/src/test/results/clientpositive/vector_nvl.q.out
+++ b/ql/src/test/results/clientpositive/vector_nvl.q.out
@@ -299,26 +299,43 @@ STAGE PLANS:
TableScan
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [13]
+ selectExpressions: ConstantVectorExpression(val null) -> 13:void
Statistics: Num rows: 12288 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out
index c9a8fe3..6768c57 100644
--- a/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out
@@ -115,13 +115,7 @@ STAGE PLANS:
TableScan
alias: b
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
Filter Operator
- Filter Vectorization:
- className: VectorFilterOperator
- native: true
- predicateExpression: SelectColumnIsNotNull(col 0:int)
predicate: key is not null (type: boolean)
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
Sorted Merge Bucket Map Join Operator
@@ -131,35 +125,21 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
outputColumnNames: _col0, _col1, _col5, _col6
- SMB Map Join Vectorization:
- className: VectorSMBMapJoinOperator
- native: false
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 1, 2, 3]
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin
+ vectorized: false
Stage: Stage-0
Fetch Operator
@@ -202,13 +182,7 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
Filter Operator
- Filter Vectorization:
- className: VectorFilterOperator
- native: true
- predicateExpression: SelectColumnIsNotNull(col 0:int)
predicate: key is not null (type: boolean)
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
Sorted Merge Bucket Map Join Operator
@@ -218,35 +192,21 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
outputColumnNames: _col0, _col1, _col5, _col6
- SMB Map Join Vectorization:
- className: VectorSMBMapJoinOperator
- native: false
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 1, 2, 3]
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin
+ vectorized: false
Stage: Stage-0
Fetch Operator
@@ -289,13 +249,7 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
- TableScan Vectorization:
- native: true
Filter Operator
- Filter Vectorization:
- className: VectorFilterOperator
- native: true
- predicateExpression: SelectColumnIsNotNull(col 0:int)
predicate: key is not null (type: boolean)
Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE
Sorted Merge Bucket Map Join Operator
@@ -305,35 +259,21 @@ STAGE PLANS:
0 key (type: int)
1 key (type: int)
outputColumnNames: _col0, _col1, _col5, _col6
- SMB Map Join Vectorization:
- className: VectorSMBMapJoinOperator
- native: false
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumnNums: [0, 1, 2, 3]
File Output Operator
compressed: false
- File Sink Vectorization:
- className: VectorFileSinkOperator
- native: false
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
- inputFormatFeatureSupport: []
- featureSupportInUse: []
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
+ notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin
+ vectorized: false
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index a498428..01bd2f6 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -47,7 +47,8 @@ public abstract class ColumnVector {
STRUCT,
LIST,
MAP,
- UNION
+ UNION,
+ VOID
}
public final Type type;
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java
new file mode 100644
index 0000000..2108a68
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+/**
+ * This class represents a void (or no) type column vector.
+ *
+ * No value vector(s) needed. Always a NULL.
+ */
+public class VoidColumnVector extends ColumnVector {
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public VoidColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len
+ */
+ public VoidColumnVector(int len) {
+ super(Type.VOID, len);
+ }
+
+ @Override
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setElement(int outputElementNum, int inputElementNum,
+ ColumnVector inputColVector) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void copySelected(boolean selectedInUse, int[] sel, int size,
+ ColumnVector outputColVector) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ throw new UnsupportedOperationException();
+ }
+}
[3/3] hive git commit: HIVE-19037: Vectorization: Miscellaneous
cleanup (Matt McCline, reviewed by Teddy Choi)
Posted by mm...@apache.org.
HIVE-19037: Vectorization: Miscellaneous cleanup (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d0f9c07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d0f9c07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d0f9c07
Branch: refs/heads/master
Commit: 9d0f9c07e50224416e5bc4c5961f032d83d446d9
Parents: fc48d72
Author: Matt McCline <mm...@hortonworks.com>
Authored: Sat Mar 31 11:01:21 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Sat Mar 31 11:01:21 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +
.../test/resources/testconfiguration.properties | 1 +
.../exec/vector/VectorExpressionDescriptor.java | 4 +-
.../ql/exec/vector/VectorizationContext.java | 301 ++-
.../ql/exec/vector/VectorizedBatchUtil.java | 70 +-
.../expressions/ConstantVectorExpression.java | 12 +
.../vector/expressions/VectorExpression.java | 12 +-
.../VectorReduceSinkCommonOperator.java | 2 -
.../hive/ql/optimizer/physical/Vectorizer.java | 91 +-
.../clientpositive/annotate_stats_select.q | 1 +
.../vector_annotate_stats_select.q | 146 ++
.../clientpositive/vector_null_projection.q | 4 +-
.../llap/vector_annotate_stats_select.q.out | 2111 ++++++++++++++++++
.../llap/vector_case_when_1.q.out | 2 +-
.../llap/vector_case_when_2.q.out | 2 +-
.../llap/vector_null_projection.q.out | 30 +-
.../clientpositive/llap/vector_nvl.q.out | 22 +-
.../llap/vector_orc_merge_incompat_schema.q.out | 10 +-
.../llap/vector_orc_nested_column_pruning.q.out | 75 +-
.../llap/vector_udf_adaptor_1.q.out | 4 +-
.../clientpositive/vector_case_when_1.q.out | 2 +-
.../clientpositive/vector_case_when_2.q.out | 2 +-
.../clientpositive/vector_null_projection.q.out | 29 +-
.../results/clientpositive/vector_nvl.q.out | 21 +-
.../vectorized_bucketmapjoin1.q.out | 72 +-
.../hive/ql/exec/vector/ColumnVector.java | 3 +-
.../hive/ql/exec/vector/VoidColumnVector.java | 65 +
27 files changed, 2855 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 697b194..02367eb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3013,6 +3013,11 @@ public class HiveConf extends Configuration {
"internal use only, used to suppress \"Execution mode: vectorized\" EXPLAIN display.\n" +
"The default is false, of course",
true),
+ HIVE_TEST_VECTORIZER_SUPPRESS_FATAL_EXCEPTIONS(
+ "hive.test.vectorizer.suppress.fatal.exceptions", true,
+ "internal use only. When false, don't suppress fatal exceptions like\n" +
+ "NullPointerException, etc so the query will fail and assure it will be noticed",
+ true),
HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, "This property has been extended to control "
+ "whether to check, convert, and normalize partition value to conform to its column type in "
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 669d0ea..516f804 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -727,6 +727,7 @@ minillaplocal.query.files=\
union_fast_stats.q,\
union_remove_26.q,\
union_top_level.q,\
+ vector_annotate_stats_select.q,\
vector_auto_smb_mapjoin_14.q,\
vector_char_varchar_1.q,\
vector_complex_all.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
index 3167e9e..2d73ab4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
@@ -80,6 +80,7 @@ public class VectorExpressionDescriptor {
DECIMAL_64 (0x1000),
LIST (0x2000),
MAP (0x4000),
+ VOID (0x8000),
INT_DECIMAL_64_FAMILY (INT_FAMILY.value | DECIMAL_64.value),
DATETIME_FAMILY (DATE.value | TIMESTAMP.value),
INTERVAL_FAMILY (INTERVAL_YEAR_MONTH.value | INTERVAL_DAY_TIME.value),
@@ -134,8 +135,7 @@ public class VectorExpressionDescriptor {
} else if (VectorizationContext.mapTypePattern.matcher(lower).matches()) {
return MAP;
} else if (lower.equals("void")) {
- // The old code let void through...
- return INT_FAMILY;
+ return VOID;
} else {
return NONE;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index e7f2b54..abbbe9a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -647,60 +647,74 @@ public class VectorizationContext {
return ocm.currentScratchColumns();
}
- private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc
- exprDesc, VectorExpressionDescriptor.Mode mode) throws HiveException {
- int columnNum = getInputColumnIndex(exprDesc.getColumn());
+ private VectorExpression getFilterOnBooleanColumnExpression(ExprNodeColumnDesc exprDesc,
+ int columnNum) throws HiveException {
VectorExpression expr = null;
- switch (mode) {
- case FILTER:
- // Evaluate the column as a boolean, converting if necessary.
- TypeInfo typeInfo = exprDesc.getTypeInfo();
- if (typeInfo.getCategory() == Category.PRIMITIVE &&
- ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.BOOLEAN) {
- expr = new SelectColumnIsTrue(columnNum);
- } else {
- // Ok, we need to convert.
- ArrayList<ExprNodeDesc> exprAsList = new ArrayList<ExprNodeDesc>(1);
- exprAsList.add(exprDesc);
-
- // First try our cast method that will handle a few special cases.
- VectorExpression castToBooleanExpr = getCastToBoolean(exprAsList);
- if (castToBooleanExpr == null) {
-
- // Ok, try the UDF.
- castToBooleanExpr = getVectorExpressionForUdf(null, UDFToBoolean.class, exprAsList,
- VectorExpressionDescriptor.Mode.PROJECTION, TypeInfoFactory.booleanTypeInfo);
- if (castToBooleanExpr == null) {
- throw new HiveException("Cannot vectorize converting expression " +
- exprDesc.getExprString() + " to boolean");
- }
- }
-
- final int outputColumnNum = castToBooleanExpr.getOutputColumnNum();
- expr = new SelectColumnIsTrue(outputColumnNum);
+ // Evaluate the column as a boolean, converting if necessary.
+ TypeInfo typeInfo = exprDesc.getTypeInfo();
+ if (typeInfo.getCategory() == Category.PRIMITIVE &&
+ ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.BOOLEAN) {
+ expr = new SelectColumnIsTrue(columnNum);
- expr.setChildExpressions(new VectorExpression[] {castToBooleanExpr});
+ expr.setInputTypeInfos(typeInfo);
+ expr.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE);
- expr.setInputTypeInfos(castToBooleanExpr.getOutputTypeInfo());
- expr.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE);
+ } else {
+ // Ok, we need to convert.
+ ArrayList<ExprNodeDesc> exprAsList = new ArrayList<ExprNodeDesc>(1);
+ exprAsList.add(exprDesc);
+
+ // First try our cast method that will handle a few special cases.
+ VectorExpression castToBooleanExpr = getCastToBoolean(exprAsList);
+ if (castToBooleanExpr == null) {
+
+ // Ok, try the UDF.
+ castToBooleanExpr = getVectorExpressionForUdf(null, UDFToBoolean.class, exprAsList,
+ VectorExpressionDescriptor.Mode.PROJECTION, TypeInfoFactory.booleanTypeInfo);
+ if (castToBooleanExpr == null) {
+ throw new HiveException("Cannot vectorize converting expression " +
+ exprDesc.getExprString() + " to boolean");
}
- break;
- case PROJECTION:
- {
- expr = new IdentityExpression(columnNum);
+ }
- TypeInfo identityTypeInfo = exprDesc.getTypeInfo();
- DataTypePhysicalVariation identityDataTypePhysicalVariation =
- getDataTypePhysicalVariation(columnNum);
+ final int outputColumnNum = castToBooleanExpr.getOutputColumnNum();
- expr.setInputTypeInfos(identityTypeInfo);
- expr.setInputDataTypePhysicalVariations(identityDataTypePhysicalVariation);
+ expr = new SelectColumnIsTrue(outputColumnNum);
- expr.setOutputTypeInfo(identityTypeInfo);
- expr.setOutputDataTypePhysicalVariation(identityDataTypePhysicalVariation);
- }
- break;
+ expr.setChildExpressions(new VectorExpression[] {castToBooleanExpr});
+
+ expr.setInputTypeInfos(castToBooleanExpr.getOutputTypeInfo());
+ expr.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE);
+ }
+ return expr;
+ }
+
+ private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc exprDesc,
+ VectorExpressionDescriptor.Mode mode) throws HiveException {
+ int columnNum = getInputColumnIndex(exprDesc.getColumn());
+ VectorExpression expr = null;
+ switch (mode) {
+ case FILTER:
+ expr = getFilterOnBooleanColumnExpression(exprDesc, columnNum);
+ break;
+ case PROJECTION:
+ {
+ expr = new IdentityExpression(columnNum);
+
+ TypeInfo identityTypeInfo = exprDesc.getTypeInfo();
+ DataTypePhysicalVariation identityDataTypePhysicalVariation =
+ getDataTypePhysicalVariation(columnNum);
+
+ expr.setInputTypeInfos(identityTypeInfo);
+ expr.setInputDataTypePhysicalVariations(identityDataTypePhysicalVariation);
+
+ expr.setOutputTypeInfo(identityTypeInfo);
+ expr.setOutputDataTypePhysicalVariation(identityDataTypePhysicalVariation);
+ }
+ break;
+ default:
+ throw new RuntimeException("Unexpected mode " + mode);
}
return expr;
}
@@ -1550,6 +1564,11 @@ public class VectorizationContext {
isColumnScaleEstablished = true;
columnScale = scale1;
} else if (child1 instanceof ExprNodeConstantDesc) {
+ if (isNullConst(child1)) {
+
+ // Cannot handle NULL scalar parameter.
+ return null;
+ }
hasScalar = true;
builder.setInputExpressionType(0, InputExpressionType.SCALAR);
} else {
@@ -1575,6 +1594,11 @@ public class VectorizationContext {
if (!isColumnScaleEstablished) {
return null;
}
+ if (isNullConst(child2)) {
+
+ // Cannot handle NULL scalar parameter.
+ return null;
+ }
hasScalar = true;
builder.setInputExpressionType(1, InputExpressionType.SCALAR);
} else {
@@ -1609,8 +1633,13 @@ public class VectorizationContext {
int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child);
if (childrenMode == VectorExpressionDescriptor.Mode.FILTER) {
- // In filter mode, the column must be a boolean
- children.add(new SelectColumnIsTrue(colIndex));
+ VectorExpression filterExpr =
+ getFilterOnBooleanColumnExpression((ExprNodeColumnDesc) child, colIndex);
+ if (filterExpr == null) {
+ return null;
+ }
+
+ children.add(filterExpr);
}
arguments[i] = colIndex;
} else {
@@ -1720,6 +1749,10 @@ public class VectorizationContext {
|| (child instanceof ExprNodeFieldDesc)) {
builder.setInputExpressionType(i, InputExpressionType.COLUMN);
} else if (child instanceof ExprNodeConstantDesc) {
+ if (isNullConst(child)) {
+ // Cannot handle NULL scalar parameter.
+ return null;
+ }
builder.setInputExpressionType(i, InputExpressionType.SCALAR);
} else if (child instanceof ExprNodeDynamicValueDesc) {
builder.setInputExpressionType(i, InputExpressionType.DYNAMICVALUE);
@@ -2408,10 +2441,26 @@ public class VectorizationContext {
private byte[] getStringScalarAsByteArray(ExprNodeConstantDesc exprNodeConstantDesc)
throws HiveException {
Object o = getScalarValue(exprNodeConstantDesc);
- if (!(o instanceof byte[])) {
- throw new HiveException("Expected constant argument of type string");
+ if (o instanceof byte[]) {
+ return (byte[]) o;
+ } else if (o instanceof HiveChar) {
+ HiveChar hiveChar = (HiveChar) o;
+ try {
+ return hiveChar.getStrippedValue().getBytes("UTF-8");
+ } catch (Exception ex) {
+ throw new HiveException(ex);
+ }
+ } else if (o instanceof HiveVarchar) {
+ HiveVarchar hiveVarchar = (HiveVarchar) o;
+ try {
+ return hiveVarchar.getValue().getBytes("UTF-8");
+ } catch (Exception ex) {
+ throw new HiveException(ex);
+ }
+ } else {
+ throw new HiveException("Expected constant argument of string family but found " +
+ o.getClass().getSimpleName());
}
- return (byte[]) o;
}
private PrimitiveCategory getAnyIntegerPrimitiveCategoryFromUdfClass(Class<? extends UDF> udfClass) {
@@ -2461,7 +2510,8 @@ public class VectorizationContext {
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
HiveDecimal rawDecimal;
- switch (ptinfo.getPrimitiveCategory()) {
+ PrimitiveCategory primitiveCategory = ptinfo.getPrimitiveCategory();
+ switch (primitiveCategory) {
case FLOAT:
rawDecimal = HiveDecimal.create(String.valueOf(scalar));
break;
@@ -2480,11 +2530,20 @@ public class VectorizationContext {
case LONG:
rawDecimal = HiveDecimal.create((Long) scalar);
break;
+ case STRING:
+ rawDecimal = HiveDecimal.create((String) scalar);
+ break;
+ case CHAR:
+ rawDecimal = HiveDecimal.create(((HiveChar) scalar).getStrippedValue());
+ break;
+ case VARCHAR:
+ rawDecimal = HiveDecimal.create(((HiveVarchar) scalar).getValue());
+ break;
case DECIMAL:
rawDecimal = (HiveDecimal) scalar;
break;
default:
- throw new HiveException("Unsupported type " + typename + " for cast to HiveDecimal");
+ throw new HiveException("Unsupported primitive category " + primitiveCategory + " for cast to HiveDecimal");
}
if (rawDecimal == null) {
if (LOG.isDebugEnabled()) {
@@ -2523,7 +2582,8 @@ public class VectorizationContext {
}
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
- switch (ptinfo.getPrimitiveCategory()) {
+ PrimitiveCategory primitiveCategory = ptinfo.getPrimitiveCategory();
+ switch (primitiveCategory) {
case FLOAT:
case DOUBLE:
case BYTE:
@@ -2531,11 +2591,17 @@ public class VectorizationContext {
case INT:
case LONG:
return ((Number) scalar).doubleValue();
+ case STRING:
+ return Double.valueOf((String) scalar);
+ case CHAR:
+ return Double.valueOf(((HiveChar) scalar).getStrippedValue());
+ case VARCHAR:
+ return Double.valueOf(((HiveVarchar) scalar).getValue());
case DECIMAL:
HiveDecimal decimalVal = (HiveDecimal) scalar;
return decimalVal.doubleValue();
default:
- throw new HiveException("Unsupported type "+typename+" for cast to Double");
+ throw new HiveException("Unsupported primitive category " + primitiveCategory + " for cast to DOUBLE");
}
}
@@ -2546,7 +2612,8 @@ public class VectorizationContext {
}
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
- switch (ptinfo.getPrimitiveCategory()) {
+ PrimitiveCategory primitiveCategory = ptinfo.getPrimitiveCategory();
+ switch (primitiveCategory) {
case FLOAT:
case DOUBLE:
case BYTE:
@@ -2554,6 +2621,45 @@ public class VectorizationContext {
case INT:
case LONG:
return ((Number) scalar).longValue();
+ case STRING:
+ case CHAR:
+ case VARCHAR:
+ {
+ final long longValue;
+ if (primitiveCategory == PrimitiveCategory.STRING) {
+ longValue = Long.valueOf((String) scalar);
+ } else if (primitiveCategory == PrimitiveCategory.CHAR) {
+ longValue = Long.valueOf(((HiveChar) scalar).getStrippedValue());
+ } else {
+ longValue = Long.valueOf(((HiveVarchar) scalar).getValue());
+ }
+ switch (integerPrimitiveCategory) {
+ case BYTE:
+ if (longValue != ((byte) longValue)) {
+ // Accurate byte value cannot be obtained.
+ return null;
+ }
+ break;
+ case SHORT:
+ if (longValue != ((short) longValue)) {
+ // Accurate short value cannot be obtained.
+ return null;
+ }
+ break;
+ case INT:
+ if (longValue != ((int) longValue)) {
+ // Accurate int value cannot be obtained.
+ return null;
+ }
+ break;
+ case LONG:
+ // No range check needed.
+ break;
+ default:
+ throw new RuntimeException("Unexpected integer primitive type " + integerPrimitiveCategory);
+ }
+ return longValue;
+ }
case DECIMAL:
HiveDecimal decimalVal = (HiveDecimal) scalar;
switch (integerPrimitiveCategory) {
@@ -2587,7 +2693,7 @@ public class VectorizationContext {
// We only store longs in our LongColumnVector.
return decimalVal.longValue();
default:
- throw new HiveException("Unsupported type "+typename+" for cast to Long");
+ throw new HiveException("Unsupported primitive category " + primitiveCategory + " for cast to LONG");
}
}
@@ -3231,6 +3337,16 @@ public class VectorizationContext {
variableArgPositions.add(i);
exprResultColumnNums.add(e.getOutputColumnNum());
argDescs[i].setVariable(e.getOutputColumnNum());
+ } else if (child instanceof ExprNodeFieldDesc) {
+ // Get the GenericUDFStructField to process the field of Struct type
+ VectorExpression e =
+ getGenericUDFStructField(
+ (ExprNodeFieldDesc) child, VectorExpressionDescriptor.Mode.PROJECTION,
+ child.getTypeInfo());
+ vectorExprs.add(e);
+ variableArgPositions.add(i);
+ exprResultColumnNums.add(e.getOutputColumnNum());
+ argDescs[i].setVariable(e.getOutputColumnNum());
} else {
throw new HiveException("Unable to vectorize custom UDF. Encountered unsupported expr desc : "
+ child);
@@ -3535,41 +3651,44 @@ public class VectorizationContext {
PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
switch (primitiveCategory) {
- case BOOLEAN:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- case DATE:
- case INTERVAL_YEAR_MONTH:
- return ColumnVector.Type.LONG;
-
- case TIMESTAMP:
- return ColumnVector.Type.TIMESTAMP;
-
- case INTERVAL_DAY_TIME:
- return ColumnVector.Type.INTERVAL_DAY_TIME;
-
- case FLOAT:
- case DOUBLE:
- return ColumnVector.Type.DOUBLE;
-
- case STRING:
- case CHAR:
- case VARCHAR:
- case BINARY:
- return ColumnVector.Type.BYTES;
-
- case DECIMAL:
- if (dataTypePhysicalVariation != null &&
- dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
- return ColumnVector.Type.DECIMAL_64;
- } else {
- return ColumnVector.Type.DECIMAL;
- }
+ case BOOLEAN:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ case DATE:
+ case INTERVAL_YEAR_MONTH:
+ return ColumnVector.Type.LONG;
+
+ case TIMESTAMP:
+ return ColumnVector.Type.TIMESTAMP;
+
+ case INTERVAL_DAY_TIME:
+ return ColumnVector.Type.INTERVAL_DAY_TIME;
+
+ case FLOAT:
+ case DOUBLE:
+ return ColumnVector.Type.DOUBLE;
+
+ case STRING:
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ return ColumnVector.Type.BYTES;
+
+ case DECIMAL:
+ if (dataTypePhysicalVariation != null &&
+ dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
+ return ColumnVector.Type.DECIMAL_64;
+ } else {
+ return ColumnVector.Type.DECIMAL;
+ }
- default:
- throw new HiveException("Unexpected primitive type category " + primitiveCategory);
+ case VOID:
+ return ColumnVector.Type.VOID;
+
+ default:
+ throw new HiveException("Unexpected primitive type category " + primitiveCategory);
}
}
default:
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 44b7c95..d92ec32 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -143,38 +143,40 @@ public class VectorizedBatchUtil {
{
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch(primitiveTypeInfo.getPrimitiveCategory()) {
- case BOOLEAN:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- case DATE:
- case INTERVAL_YEAR_MONTH:
- return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case TIMESTAMP:
- return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case INTERVAL_DAY_TIME:
- return new IntervalDayTimeColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case FLOAT:
- case DOUBLE:
- return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case BINARY:
- case STRING:
- case CHAR:
- case VARCHAR:
- return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
- case DECIMAL:
- DecimalTypeInfo tInfo = (DecimalTypeInfo) primitiveTypeInfo;
- if (dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
- return new Decimal64ColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- tInfo.precision(), tInfo.scale());
- } else {
- return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- tInfo.precision(), tInfo.scale());
- }
- default:
- throw new RuntimeException("Vectorizaton is not supported for datatype:"
- + primitiveTypeInfo.getPrimitiveCategory());
+ case BOOLEAN:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ case DATE:
+ case INTERVAL_YEAR_MONTH:
+ return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case TIMESTAMP:
+ return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case INTERVAL_DAY_TIME:
+ return new IntervalDayTimeColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case FLOAT:
+ case DOUBLE:
+ return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case BINARY:
+ case STRING:
+ case CHAR:
+ case VARCHAR:
+ return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ case DECIMAL:
+ DecimalTypeInfo tInfo = (DecimalTypeInfo) primitiveTypeInfo;
+ if (dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
+ return new Decimal64ColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ tInfo.precision(), tInfo.scale());
+ } else {
+ return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ tInfo.precision(), tInfo.scale());
+ }
+ case VOID:
+ return new VoidColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ default:
+ throw new RuntimeException("Vectorizaton is not supported for datatype:"
+ + primitiveTypeInfo.getPrimitiveCategory());
}
}
case STRUCT:
@@ -791,7 +793,9 @@ public class VectorizedBatchUtil {
case LIST:
case MAP:
case UNION:
- // No complex type support for now.
+ throw new RuntimeException("No complex type support: " + sourceColVector.type);
+ case VOID:
+ break;
default:
throw new RuntimeException("Unexpected column vector type " + sourceColVector.type);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
index 5b89131..4cc1be5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
@@ -202,6 +202,13 @@ public class ConstantVectorExpression extends VectorExpression {
}
}
+ private void evaluateVoid(VectorizedRowBatch vrg) {
+ VoidColumnVector voidColVector = (VoidColumnVector) vrg.cols[outputColumnNum];
+ voidColVector.isRepeating = true;
+ voidColVector.isNull[0] = true;
+ voidColVector.noNulls = false;
+ }
+
@Override
public void evaluate(VectorizedRowBatch vrg) {
switch (type) {
@@ -223,6 +230,11 @@ public class ConstantVectorExpression extends VectorExpression {
case INTERVAL_DAY_TIME:
evaluateIntervalDayTime(vrg);
break;
+ case VOID:
+ evaluateVoid(vrg);
+ break;
+ default:
+ throw new RuntimeException("Unexpected column vector type " + type);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java
index 4407961..3d0ee50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java
@@ -285,20 +285,18 @@ public abstract class VectorExpression implements Serializable {
}
protected String getParamTypeString(int typeNum) {
- if (inputTypeInfos == null || inputDataTypePhysicalVariations == null) {
- fake++;
+ if (inputTypeInfos == null) {
+ return "<input types is null>";
}
- if (typeNum >= inputTypeInfos.length || typeNum >= inputDataTypePhysicalVariations.length) {
- fake++;
+ if (inputDataTypePhysicalVariations == null) {
+ return "<input data type physical variations is null>";
}
return getTypeName(inputTypeInfos[typeNum], inputDataTypePhysicalVariations[typeNum]);
}
- static int fake;
-
public static String getTypeName(TypeInfo typeInfo, DataTypePhysicalVariation dataTypePhysicalVariation) {
if (typeInfo == null) {
- fake++;
+ return "<input type is null>";
}
if (dataTypePhysicalVariation != null && dataTypePhysicalVariation != DataTypePhysicalVariation.NONE) {
return typeInfo.toString() + "/" + dataTypePhysicalVariation;
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java
index cc876d5..4077552 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java
@@ -154,8 +154,6 @@ public abstract class VectorReduceSinkCommonOperator extends TerminalOperator<Re
VectorizationContext vContext, VectorDesc vectorDesc) throws HiveException {
this(ctx);
- LOG.info("VectorReduceSinkCommonOperator constructor");
-
ReduceSinkDesc desc = (ReduceSinkDesc) conf;
this.conf = desc;
this.vContext = vContext;
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index a822a4b..d3fbf07 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -320,6 +320,7 @@ public class Vectorizer implements PhysicalPlanResolver {
private Collection<Class<?>> rowDeserializeInputFormatExcludes;
private int vectorizedPTFMaxMemoryBufferingBatchCount;
private int vectorizedTestingReducerBatchSize;
+ private boolean isTestVectorizerSuppressFatalExceptions;
private boolean isSchemaEvolution;
@@ -1847,6 +1848,25 @@ public class Vectorizer implements PhysicalPlanResolver {
// The "not vectorized" information has been stored in the MapWork vertex.
return false;
+ } catch (NullPointerException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ // Re-throw without losing original stack trace.
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
+ } catch (ClassCastException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
+ } catch (RuntimeException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
}
vectorTaskColumnInfo.setNeededVirtualColumnList(
@@ -2020,6 +2040,25 @@ public class Vectorizer implements PhysicalPlanResolver {
// The "not vectorized" information has been stored in the MapWork vertex.
return false;
+ } catch (NullPointerException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ // Re-throw without losing original stack trace.
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
+ } catch (ClassCastException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
+ } catch (RuntimeException e) {
+ if (!isTestVectorizerSuppressFatalExceptions) {
+ throw e;
+ }
+ setNodeIssue("exception: " + VectorizationContext.getStackTraceAsSingleLine(e));
+ return false;
}
/*
@@ -2226,6 +2265,10 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveConf.getIntVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_TESTING_REDUCER_BATCH_SIZE);
+ isTestVectorizerSuppressFatalExceptions =
+ HiveConf.getBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVE_TEST_VECTORIZER_SUPPRESS_FATAL_EXCEPTIONS);
+
vectorizedInputFormatSupportEnabled =
HiveConf.getVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED);
@@ -2362,7 +2405,11 @@ public class Vectorizer implements PhysicalPlanResolver {
private boolean validateSelectOperator(SelectOperator op) {
List<ExprNodeDesc> descList = op.getConf().getColList();
for (ExprNodeDesc desc : descList) {
- boolean ret = validateExprNodeDesc(desc, "Select");
+ boolean ret =
+ validateExprNodeDesc(
+ desc, "Select",
+ VectorExpressionDescriptor.Mode.PROJECTION,
+ /* allowComplex */ true, /* allowVoidProjection */ true);
if (!ret) {
return false;
}
@@ -2737,6 +2784,12 @@ public class Vectorizer implements PhysicalPlanResolver {
private boolean validateExprNodeDescRecursive(ExprNodeDesc desc, String expressionTitle,
VectorExpressionDescriptor.Mode mode, boolean allowComplex) {
+ return validateExprNodeDescRecursive(desc, expressionTitle, mode,
+ allowComplex, /* allowVoidProjection */ false);
+ }
+
+ private boolean validateExprNodeDescRecursive(ExprNodeDesc desc, String expressionTitle,
+ VectorExpressionDescriptor.Mode mode, boolean allowComplex, boolean allowVoidProjection) {
if (desc instanceof ExprNodeColumnDesc) {
ExprNodeColumnDesc c = (ExprNodeColumnDesc) desc;
String columnName = c.getColumn();
@@ -2760,7 +2813,8 @@ public class Vectorizer implements PhysicalPlanResolver {
}
}
String typeName = desc.getTypeInfo().getTypeName();
- boolean ret = validateDataType(typeName, mode, allowComplex && isVectorizationComplexTypesEnabled);
+ boolean ret = validateDataType(
+ typeName, mode, allowComplex && isVectorizationComplexTypesEnabled, allowVoidProjection);
if (!ret) {
setExpressionIssue(expressionTitle,
getValidateDataTypeErrorMsg(
@@ -2783,7 +2837,8 @@ public class Vectorizer implements PhysicalPlanResolver {
&& desc.getChildren().get(0).getTypeInfo().getCategory() == Category.STRUCT) {
// Don't restrict child expressions for projection.
// Always use loose FILTER mode.
- if (!validateStructInExpression(desc, expressionTitle, VectorExpressionDescriptor.Mode.FILTER)) {
+ if (!validateStructInExpression(
+ desc, expressionTitle, VectorExpressionDescriptor.Mode.FILTER)) {
return false;
}
} else {
@@ -2791,7 +2846,8 @@ public class Vectorizer implements PhysicalPlanResolver {
// Don't restrict child expressions for projection.
// Always use loose FILTER mode.
if (!validateExprNodeDescRecursive(
- d, expressionTitle, VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true)) {
+ d, expressionTitle, VectorExpressionDescriptor.Mode.FILTER,
+ /* allowComplex */ true, allowVoidProjection)) {
return false;
}
}
@@ -2843,7 +2899,8 @@ public class Vectorizer implements PhysicalPlanResolver {
private boolean validateExprNodeDesc(ExprNodeDesc desc, String expressionTitle) {
return validateExprNodeDesc(
- desc, expressionTitle, VectorExpressionDescriptor.Mode.PROJECTION, /* allowComplex */ true);
+ desc, expressionTitle, VectorExpressionDescriptor.Mode.PROJECTION,
+ /* allowComplex */ true, /* allowVoidProjection */ false);
}
boolean validateExprNodeDesc(ExprNodeDesc desc, String expressionTitle,
@@ -2851,6 +2908,12 @@ public class Vectorizer implements PhysicalPlanResolver {
return validateExprNodeDescRecursive(desc, expressionTitle, mode, allowComplex);
}
+ boolean validateExprNodeDesc(ExprNodeDesc desc, String expressionTitle,
+ VectorExpressionDescriptor.Mode mode, boolean allowComplex, boolean allowVoidProjection) {
+ return validateExprNodeDescRecursive(
+ desc, expressionTitle, mode, allowComplex, allowVoidProjection);
+ }
+
private boolean validateGenericUdf(ExprNodeGenericFuncDesc genericUDFExpr) {
if (VectorizationContext.isCustomUDF(genericUDFExpr)) {
return true;
@@ -2895,10 +2958,16 @@ public class Vectorizer implements PhysicalPlanResolver {
public static boolean validateDataType(String type, VectorExpressionDescriptor.Mode mode,
boolean allowComplex) {
+ return validateDataType(type, mode, allowComplex, /* allowVoidProjection */ false);
+ }
+
+ public static boolean validateDataType(String type, VectorExpressionDescriptor.Mode mode,
+ boolean allowComplex, boolean allowVoidProjection) {
type = type.toLowerCase();
boolean result = supportedDataTypesPattern.matcher(type).matches();
- if (result && mode == VectorExpressionDescriptor.Mode.PROJECTION && type.equals("void")) {
+ if (result && !allowVoidProjection &&
+ mode == VectorExpressionDescriptor.Mode.PROJECTION && type.equals("void")) {
return false;
}
@@ -2915,10 +2984,18 @@ public class Vectorizer implements PhysicalPlanResolver {
public static String getValidateDataTypeErrorMsg(String type, VectorExpressionDescriptor.Mode mode,
boolean allowComplex, boolean isVectorizationComplexTypesEnabled) {
+ return getValidateDataTypeErrorMsg(
+ type, mode, allowComplex, isVectorizationComplexTypesEnabled, false);
+ }
+
+ public static String getValidateDataTypeErrorMsg(String type, VectorExpressionDescriptor.Mode mode,
+ boolean allowComplex, boolean isVectorizationComplexTypesEnabled,
+ boolean allowVoidProjection) {
type = type.toLowerCase();
boolean result = supportedDataTypesPattern.matcher(type).matches();
- if (result && mode == VectorExpressionDescriptor.Mode.PROJECTION && type.equals("void")) {
+ if (result && !allowVoidProjection &&
+ mode == VectorExpressionDescriptor.Mode.PROJECTION && type.equals("void")) {
return "Vectorizing data type void not supported when mode = PROJECTION";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/queries/clientpositive/annotate_stats_select.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/annotate_stats_select.q b/ql/src/test/queries/clientpositive/annotate_stats_select.q
index 5aaff11..3921f90 100644
--- a/ql/src/test/queries/clientpositive/annotate_stats_select.q
+++ b/ql/src/test/queries/clientpositive/annotate_stats_select.q
@@ -1,3 +1,4 @@
+SET hive.vectorized.execution.enabled=false;
set hive.mapred.mode=nonstrict;
set hive.stats.fetch.column.stats=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/queries/clientpositive/vector_annotate_stats_select.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_annotate_stats_select.q b/ql/src/test/queries/clientpositive/vector_annotate_stats_select.q
new file mode 100644
index 0000000..e45dff6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_annotate_stats_select.q
@@ -0,0 +1,146 @@
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+set hive.mapred.mode=nonstrict;
+set hive.stats.fetch.column.stats=true;
+
+create table if not exists alltypes (
+ bo1 boolean,
+ ti1 tinyint,
+ si1 smallint,
+ i1 int,
+ bi1 bigint,
+ f1 float,
+ d1 double,
+ de1 decimal,
+ ts1 timestamp,
+ da1 timestamp,
+ s1 string,
+ vc1 varchar(5),
+ m1 map<string, string>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:string>
+) row format delimited fields terminated by '|'
+collection items terminated by ','
+map keys terminated by ':' stored as textfile;
+
+create table alltypes_orc like alltypes;
+alter table alltypes_orc set fileformat orc;
+
+load data local inpath '../../data/files/alltypes.txt' overwrite into table alltypes;
+
+insert overwrite table alltypes_orc select * from alltypes;
+
+-- basicStatState: COMPLETE colStatState: NONE numRows: 2 rawDataSize: 1514
+explain select * from alltypes_orc;
+
+-- statistics for complex types are not supported yet
+analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1;
+
+-- numRows: 2 rawDataSize: 1514
+explain select * from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 8
+explain select bo1 from alltypes_orc;
+
+-- col alias renaming
+-- numRows: 2 rawDataSize: 8
+explain select i1 as int1 from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 174
+explain select s1 from alltypes_orc;
+
+-- column statistics for complex types unsupported and so statistics will not be updated
+-- numRows: 2 rawDataSize: 1514
+explain select m1 from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 246
+explain select bo1, ti1, si1, i1, bi1, f1, d1,s1 from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 0
+explain vectorization expression select null from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 8
+explain vectorization expression select 11 from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 16
+explain vectorization expression select 11L from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 16
+explain vectorization expression select 11.0 from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 178
+explain vectorization expression select "hello" from alltypes_orc;
+explain vectorization expression select cast("hello" as char(5)) from alltypes_orc;
+explain vectorization expression select cast("hello" as varchar(5)) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 96
+explain vectorization expression select unbase64("0xe23") from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 16
+explain vectorization expression select cast("1" as TINYINT), cast("20" as SMALLINT) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 80
+explain vectorization expression select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 112
+explain vectorization expression select cast("1970-12-31 15:59:58.174" as DATE) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 224
+explain vectorization expression select cast("58.174" as DECIMAL) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 112
+explain vectorization expression select array(1,2,3) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 1508
+explain vectorization expression select str_to_map("a=1 b=2 c=3", " ", "=") from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 112
+explain vectorization expression select NAMED_STRUCT("a", 11, "b", 11) from alltypes_orc;
+
+-- numRows: 2 rawDataSize: 250
+explain vectorization expression select CREATE_UNION(0, "hello") from alltypes_orc;
+
+-- COUNT(*) is projected as new column. It is not projected as GenericUDF and so datasize estimate will be based on number of rows
+-- numRows: 1 rawDataSize: 8
+explain vectorization expression select count(*) from alltypes_orc;
+
+-- COUNT(1) is projected as new column. It is not projected as GenericUDF and so datasize estimate will be based on number of rows
+-- numRows: 1 rawDataSize: 8
+explain vectorization expression select count(1) from alltypes_orc;
+
+-- column statistics for complex column types will be missing. data size will be calculated from available column statistics
+-- numRows: 2 rawDataSize: 254
+explain vectorization expression select *,11 from alltypes_orc;
+
+-- subquery selects
+-- inner select - numRows: 2 rawDataSize: 8
+-- outer select - numRows: 2 rawDataSize: 8
+explain vectorization expression select i1 from (select i1 from alltypes_orc limit 10) temp;
+
+-- inner select - numRows: 2 rawDataSize: 16
+-- outer select - numRows: 2 rawDataSize: 8
+explain vectorization expression select i1 from (select i1,11 from alltypes_orc limit 10) temp;
+
+-- inner select - numRows: 2 rawDataSize: 16
+-- outer select - numRows: 2 rawDataSize: 186
+explain vectorization expression select i1,"hello" from (select i1,11 from alltypes_orc limit 10) temp;
+
+-- inner select - numRows: 2 rawDataSize: 24
+-- outer select - numRows: 2 rawDataSize: 16
+explain vectorization expression select x from (select i1,11.0 as x from alltypes_orc limit 10) temp;
+
+-- inner select - numRows: 2 rawDataSize: 104
+-- outer select - numRows: 2 rawDataSize: 186
+explain vectorization expression select x,"hello" from (select i1 as x, unbase64("0xe23") as ub from alltypes_orc limit 10) temp;
+
+-- inner select - numRows: 2 rawDataSize: 186
+-- middle select - numRows: 2 rawDataSize: 178
+-- outer select - numRows: 2 rawDataSize: 194
+explain vectorization expression select h, 11.0 from (select hell as h from (select i1, "hello" as hell from alltypes_orc limit 10) in1 limit 10) in2;
+
+-- This test is for FILTER operator where filter expression is a boolean column
+-- numRows: 2 rawDataSize: 8
+explain vectorization expression select bo1 from alltypes_orc where bo1;
+
+-- numRows: 0 rawDataSize: 0
+explain vectorization expression select bo1 from alltypes_orc where !bo1;
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/queries/clientpositive/vector_null_projection.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_null_projection.q b/ql/src/test/queries/clientpositive/vector_null_projection.q
index 711b8e7..6cea1e6 100644
--- a/ql/src/test/queries/clientpositive/vector_null_projection.q
+++ b/ql/src/test/queries/clientpositive/vector_null_projection.q
@@ -9,8 +9,8 @@ create table b(s string) stored as orc;
insert into table a values('aaa');
insert into table b values('aaa');
--- We expect no vectorization due to NULL (void) projection type.
-explain vectorization expression
+-- We expect some vectorization due to NULL (void) projection type.
+explain vectorization detail
select NULL from a;
select NULL from a;
[2/3] hive git commit: HIVE-19037: Vectorization: Miscellaneous
cleanup (Matt McCline, reviewed by Teddy Choi)
Posted by mm...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_annotate_stats_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_annotate_stats_select.q.out b/ql/src/test/results/clientpositive/llap/vector_annotate_stats_select.q.out
new file mode 100644
index 0000000..5613b6d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_annotate_stats_select.q.out
@@ -0,0 +1,2111 @@
+PREHOOK: query: create table if not exists alltypes (
+ bo1 boolean,
+ ti1 tinyint,
+ si1 smallint,
+ i1 int,
+ bi1 bigint,
+ f1 float,
+ d1 double,
+ de1 decimal,
+ ts1 timestamp,
+ da1 timestamp,
+ s1 string,
+ vc1 varchar(5),
+ m1 map<string, string>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:string>
+) row format delimited fields terminated by '|'
+collection items terminated by ','
+map keys terminated by ':' stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypes
+POSTHOOK: query: create table if not exists alltypes (
+ bo1 boolean,
+ ti1 tinyint,
+ si1 smallint,
+ i1 int,
+ bi1 bigint,
+ f1 float,
+ d1 double,
+ de1 decimal,
+ ts1 timestamp,
+ da1 timestamp,
+ s1 string,
+ vc1 varchar(5),
+ m1 map<string, string>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:string>
+) row format delimited fields terminated by '|'
+collection items terminated by ','
+map keys terminated by ':' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypes
+PREHOOK: query: create table alltypes_orc like alltypes
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypes_orc
+POSTHOOK: query: create table alltypes_orc like alltypes
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypes_orc
+PREHOOK: query: alter table alltypes_orc set fileformat orc
+PREHOOK: type: ALTERTABLE_FILEFORMAT
+PREHOOK: Input: default@alltypes_orc
+PREHOOK: Output: default@alltypes_orc
+POSTHOOK: query: alter table alltypes_orc set fileformat orc
+POSTHOOK: type: ALTERTABLE_FILEFORMAT
+POSTHOOK: Input: default@alltypes_orc
+POSTHOOK: Output: default@alltypes_orc
+PREHOOK: query: load data local inpath '../../data/files/alltypes.txt' overwrite into table alltypes
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@alltypes
+POSTHOOK: query: load data local inpath '../../data/files/alltypes.txt' overwrite into table alltypes
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@alltypes
+PREHOOK: query: insert overwrite table alltypes_orc select * from alltypes
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypes
+PREHOOK: Output: default@alltypes_orc
+POSTHOOK: query: insert overwrite table alltypes_orc select * from alltypes
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypes
+POSTHOOK: Output: default@alltypes_orc
+POSTHOOK: Lineage: alltypes_orc.bi1 SIMPLE [(alltypes)alltypes.FieldSchema(name:bi1, type:bigint, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.bo1 SIMPLE [(alltypes)alltypes.FieldSchema(name:bo1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.d1 SIMPLE [(alltypes)alltypes.FieldSchema(name:d1, type:double, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.da1 SIMPLE [(alltypes)alltypes.FieldSchema(name:da1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.de1 SIMPLE [(alltypes)alltypes.FieldSchema(name:de1, type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.f1 SIMPLE [(alltypes)alltypes.FieldSchema(name:f1, type:float, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.i1 SIMPLE [(alltypes)alltypes.FieldSchema(name:i1, type:int, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.l1 SIMPLE [(alltypes)alltypes.FieldSchema(name:l1, type:array<int>, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.m1 SIMPLE [(alltypes)alltypes.FieldSchema(name:m1, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.s1 SIMPLE [(alltypes)alltypes.FieldSchema(name:s1, type:string, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.si1 SIMPLE [(alltypes)alltypes.FieldSchema(name:si1, type:smallint, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.st1 SIMPLE [(alltypes)alltypes.FieldSchema(name:st1, type:struct<c1:int,c2:string>, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.ti1 SIMPLE [(alltypes)alltypes.FieldSchema(name:ti1, type:tinyint, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.ts1 SIMPLE [(alltypes)alltypes.FieldSchema(name:ts1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: alltypes_orc.vc1 SIMPLE [(alltypes)alltypes.FieldSchema(name:vc1, type:varchar(5), comment:null), ]
+PREHOOK: query: explain select * from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 3554 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: bo1 (type: boolean), ti1 (type: tinyint), si1 (type: smallint), i1 (type: int), bi1 (type: bigint), f1 (type: float), d1 (type: double), de1 (type: decimal(10,0)), ts1 (type: timestamp), da1 (type: timestamp), s1 (type: string), vc1 (type: varchar(5)), m1 (type: map<string,string>), l1 (type: array<int>), st1 (type: struct<c1:int,c2:string>)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 2 Data size: 3554 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 3554 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypes_orc
+PREHOOK: Output: default@alltypes_orc
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypes_orc
+POSTHOOK: Output: default@alltypes_orc
+#### A masked pattern was here ####
+PREHOOK: query: explain select * from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 3356 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: bo1 (type: boolean), ti1 (type: tinyint), si1 (type: smallint), i1 (type: int), bi1 (type: bigint), f1 (type: float), d1 (type: double), de1 (type: decimal(10,0)), ts1 (type: timestamp), da1 (type: timestamp), s1 (type: string), vc1 (type: varchar(5)), m1 (type: map<string,string>), l1 (type: array<int>), st1 (type: struct<c1:int,c2:string>)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 2 Data size: 3356 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 3356 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select bo1 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select bo1 from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: bo1 (type: boolean)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select i1 as int1 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select i1 as int1 from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: i1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select s1 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select s1 from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: s1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select m1 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select m1 from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1840 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: m1 (type: map<string,string>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 1840 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 1840 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select bo1, ti1, si1, i1, bi1, f1, d1,s1 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select bo1, ti1, si1, i1, bi1, f1, d1,s1 from alltypes_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 246 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: bo1 (type: boolean), ti1 (type: tinyint), si1 (type: smallint), i1 (type: int), bi1 (type: bigint), f1 (type: float), d1 (type: double), s1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Statistics: Num rows: 2 Data size: 246 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 246 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select null from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select null from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: null (type: void)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val null) -> 16:void
+ Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select 11 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select 11 from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 11 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 11) -> 16:int
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select 11L from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select 11L from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 11L (type: bigint)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 11) -> 16:bigint
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select 11.0 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select 11.0 from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 11 (type: decimal(2,0))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 11) -> 16:decimal(2,0)
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select "hello" from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select "hello" from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 'hello' (type: string)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val hello) -> 16:string
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("hello" as char(5)) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("hello" as char(5)) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 'hello' (type: char(5))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val hello) -> 16:char(5)
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("hello" as varchar(5)) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("hello" as varchar(5)) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 'hello' (type: varchar(5))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val hello) -> 16:varchar(5)
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select unbase64("0xe23") from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select unbase64("0xe23") from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: D317B6 (type: binary)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ notVectorizedReason: SELECT operator: Unsupported constant type: binary, object class byte[]
+ vectorized: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("1" as TINYINT), cast("20" as SMALLINT) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("1" as TINYINT), cast("20" as SMALLINT) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 1Y (type: tinyint), 20S (type: smallint)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16, 17]
+ selectExpressions: ConstantVectorExpression(val 1) -> 16:tinyint, ConstantVectorExpression(val 20) -> 17:smallint
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: TIMESTAMP'1970-12-31 15:59:58.174' (type: timestamp)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 1970-12-31 15:59:58.174) -> 16:timestamp
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("1970-12-31 15:59:58.174" as DATE) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("1970-12-31 15:59:58.174" as DATE) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: DATE'1970-12-31' (type: date)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 364) -> 16:date
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select cast("58.174" as DECIMAL) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select cast("58.174" as DECIMAL) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: 58 (type: decimal(10,0))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: ConstantVectorExpression(val 58) -> 16:decimal(10,0)
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select array(1,2,3) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select array(1,2,3) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: array(1,2,3) (type: array<int>)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: VectorUDFAdaptor(array(1,2,3)) -> 16:array<int>
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select str_to_map("a=1 b=2 c=3", " ", "=") from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select str_to_map("a=1 b=2 c=3", " ", "=") from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: str_to_map('a=1 b=2 c=3',' ','=') (type: map<string,string>)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: VectorUDFAdaptor(str_to_map('a=1 b=2 c=3',' ','=')) -> 16:map<string,string>
+ Statistics: Num rows: 2 Data size: 1508 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 1508 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select NAMED_STRUCT("a", 11, "b", 11) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select NAMED_STRUCT("a", 11, "b", 11) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: named_struct('a',11,'b',11) (type: struct<a:int,b:int>)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: VectorUDFAdaptor(named_struct('a',11,'b',11)) -> 16:struct<a:int,b:int>
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select CREATE_UNION(0, "hello") from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select CREATE_UNION(0, "hello") from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: create_union(0,'hello') (type: uniontype<string>)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [16]
+ selectExpressions: VectorUDFAdaptor(create_union(0,'hello')) -> 16:uniontype<string>
+ Statistics: Num rows: 2 Data size: 250 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 250 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select count(*) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select count(*) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select count(1) from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select count(1) from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select *,11 from alltypes_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select *,11 from alltypes_orc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 3356 Basic stats: COMPLETE Column stats: PARTIAL
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: bo1 (type: boolean), ti1 (type: tinyint), si1 (type: smallint), i1 (type: int), bi1 (type: bigint), f1 (type: float), d1 (type: double), de1 (type: decimal(10,0)), ts1 (type: timestamp), da1 (type: timestamp), s1 (type: string), vc1 (type: varchar(5)), m1 (type: map<string,string>), l1 (type: array<int>), st1 (type: struct<c1:int,c2:string>), 11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16]
+ selectExpressions: ConstantVectorExpression(val 11) -> 16:int
+ Statistics: Num rows: 2 Data size: 3364 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 3364 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select i1 from (select i1 from alltypes_orc limit 10) temp
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select i1 from (select i1 from alltypes_orc limit 10) temp
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: i1 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select i1 from (select i1,11 from alltypes_orc limit 10) temp
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select i1 from (select i1,11 from alltypes_orc limit 10) temp
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: i1 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select i1,"hello" from (select i1,11 from alltypes_orc limit 10) temp
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select i1,"hello" from (select i1,11 from alltypes_orc limit 10) temp
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: i1 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkEmptyKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), 'hello' (type: string)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ selectExpressions: ConstantVectorExpression(val hello) -> 1:string
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select x from (select i1,11.0 as x from alltypes_orc limit 10) temp
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select x from (select i1,11.0 as x from alltypes_orc limit 10) temp
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: []
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkEmptyKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 11 (type: decimal(2,0))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0]
+ selectExpressions: ConstantVectorExpression(val 11) -> 0:decimal(2,0)
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select x,"hello" from (select i1 as x, unbase64("0xe23") as ub from alltypes_orc limit 10) temp
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select x,"hello" from (select i1 as x, unbase64("0xe23") as ub from alltypes_orc limit 10) temp
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ expressions: i1 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkEmptyKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0]
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), 'hello' (type: string)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ selectExpressions: ConstantVectorExpression(val hello) -> 1:string
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select h, 11.0 from (select hell as h from (select i1, "hello" as hell from alltypes_orc limit 10) in1 limit 10) in2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select h, 11.0 from (select hell as h from (select i1, "hello" as hell from alltypes_orc limit 10) in1 limit 10) in2
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ Select Operator
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: []
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkEmptyKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Limit
+ Number of rows: 10
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 'hello' (type: string), 11 (type: decimal(2,0))
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1]
+ selectExpressions: ConstantVectorExpression(val hello) -> 0:string, ConstantVectorExpression(val 11) -> 1:decimal(2,0)
+ Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 2 Data size: 402 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain vectorization expression select bo1 from alltypes_orc where bo1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression select bo1 from alltypes_orc where bo1
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+
<TRUNCATED>