You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/03/19 19:30:36 UTC
hive git commit: HIVE-16110: Vectorization: Support 2 Value CASE WHEN
instead of fall back to VectorUDFAdaptor (Matt McCline,
reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 7fc60962f -> cd4fcefb8
HIVE-16110: Vectorization: Support 2 Value CASE WHEN instead of fall back to VectorUDFAdaptor (Matt McCline, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cd4fcefb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cd4fcefb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cd4fcefb
Branch: refs/heads/master
Commit: cd4fcefb83c0151d213ed9ec1a9238ed7fc6b683
Parents: 7fc6096
Author: Matt McCline <mm...@hortonworks.com>
Authored: Sun Mar 19 14:30:29 2017 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Sun Mar 19 14:30:29 2017 -0500
----------------------------------------------------------------------
.../ql/exec/vector/VectorizationContext.java | 50 ++++
.../vector_groupby_grouping_id3.q | 4 +-
.../queries/clientpositive/vectorized_case.q | 18 ++
.../llap/vector_between_columns.q.out | 2 +-
.../llap/vector_groupby_grouping_id3.q.out | 122 ++++++++-
.../clientpositive/llap/vectorized_case.q.out | 256 +++++++++++++++++++
.../clientpositive/spark/vectorized_case.q.out | 252 ++++++++++++++++++
.../clientpositive/vector_between_columns.q.out | 2 +-
.../clientpositive/vectorized_case.q.out | 222 ++++++++++++++++
9 files changed, 919 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 2e27fd5..c3940cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1461,6 +1461,8 @@ public class VectorizationContext {
ve = getBetweenFilterExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFIn) {
ve = getInExpression(childExpr, mode, returnType);
+ } else if (udf instanceof GenericUDFWhen) {
+ ve = getWhenExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFOPPositive) {
ve = getIdentityExpression(childExpr);
} else if (udf instanceof GenericUDFCoalesce || udf instanceof GenericUDFNvl) {
@@ -2320,6 +2322,54 @@ public class VectorizationContext {
return createVectorExpression(cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
+ private boolean isColumnOrNonNullConst(ExprNodeDesc exprNodeDesc) {
+ if (exprNodeDesc instanceof ExprNodeColumnDesc) {
+ return true;
+ }
+ if (exprNodeDesc instanceof ExprNodeConstantDesc) {
+ String typeString = exprNodeDesc.getTypeString();
+ if (!typeString.equalsIgnoreCase("void")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private VectorExpression getWhenExpression(List<ExprNodeDesc> childExpr,
+ VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
+
+ if (mode != VectorExpressionDescriptor.Mode.PROJECTION) {
+ return null;
+ }
+ if (childExpr.size() != 3) {
+ // For now, we only optimize the 2 value case.
+ return null;
+ }
+
+ /*
+ * When we have 2 simple values:
+ * CASE WHEN boolExpr THEN column | const ELSE column | const END
+ * then we can convert to: IF (boolExpr THEN column | const ELSE column | const)
+ */
+ // CONSIDER: Adding a version of IfExpr* than can handle a non-column/const expression in the
+ // THEN or ELSE.
+ ExprNodeDesc exprNodeDesc1 = childExpr.get(1);
+ ExprNodeDesc exprNodeDesc2 = childExpr.get(2);
+ if (isColumnOrNonNullConst(exprNodeDesc1) &&
+ isColumnOrNonNullConst(exprNodeDesc2)) {
+ // Yes.
+ GenericUDFIf genericUDFIf = new GenericUDFIf();
+ return
+ getVectorExpressionForUdf(
+ genericUDFIf,
+ GenericUDFIf.class,
+ childExpr,
+ mode,
+ returnType);
+ }
+ return null; // Not handled by vector classes yet.
+ }
+
/*
* Return vector expression for a custom (i.e. not built-in) UDF.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/queries/clientpositive/vector_groupby_grouping_id3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby_grouping_id3.q b/ql/src/test/queries/clientpositive/vector_groupby_grouping_id3.q
index 29e9211..732b2d8 100644
--- a/ql/src/test/queries/clientpositive/vector_groupby_grouping_id3.q
+++ b/ql/src/test/queries/clientpositive/vector_groupby_grouping_id3.q
@@ -14,7 +14,7 @@ set hive.cbo.enable = false;
-- SORT_QUERY_RESULTS
-EXPLAIN
+EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
@@ -28,7 +28,7 @@ HAVING GROUPING__ID = 1;
set hive.cbo.enable = true;
-EXPLAIN
+EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/queries/clientpositive/vectorized_case.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_case.q b/ql/src/test/queries/clientpositive/vectorized_case.q
index 2efacb4..2f6810f 100644
--- a/ql/src/test/queries/clientpositive/vectorized_case.q
+++ b/ql/src/test/queries/clientpositive/vectorized_case.q
@@ -55,3 +55,21 @@ where csmallint = 418
or csmallint = 12205
or csmallint = 10583
;
+explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc;
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc;
+explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc;
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc;
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
index a58b1b2..8897163 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out
@@ -121,7 +121,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumns: [0, 2, 1, 3, 5]
- selectExpressions: VectorUDFAdaptor(CASE WHEN (_col1 BETWEEN _col3 AND _col3) THEN ('Ok') ELSE ('NoOk') END)(children: VectorUDFAdaptor(_col1 BETWEEN _col3 AND _col3) -> 4:boolean) -> 5:string
+ selectExpressions: IfExprStringScalarStringScalar(col 4, val Ok, val NoOk)(children: VectorUDFAdaptor(_col1 BETWEEN _col3 AND _col3) -> 4:boolean) -> 5:String
Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out
index 8541d12..fcf798e 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id3.q.out
@@ -27,14 +27,14 @@ POSTHOOK: Output: default@T1
POSTHOOK: Lineage: t1.key SIMPLE [(t1_text)t1_text.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: t1.value SIMPLE [(t1_text)t1_text.FieldSchema(name:value, type:int, comment:null), ]
t1_text.key t1_text.value
-PREHOOK: query: EXPLAIN
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
GROUPING SETS ((), (key))
HAVING GROUPING__ID = 1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
@@ -42,6 +42,10 @@ GROUPING SETS ((), (key))
HAVING GROUPING__ID = 1
POSTHOOK: type: QUERY
Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -59,29 +63,71 @@ STAGE PLANS:
TableScan
alias: t1
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
Select Operator
expressions: key (type: int), value (type: int)
outputColumnNames: key, value
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1]
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
+ Group By Vectorization:
+ aggregators: VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0, col 1, ConstantVectorExpression(val 0) -> 2:long
+ native: false
+ projectedOutputColumns: [0]
keys: key (type: int), value (type: int), 0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 12 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterLongColEqualLongScalar(col 2, val 1) -> boolean
predicate: (_col2 = 1) (type: boolean)
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int), 1 (type: int)
sort order: +++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int), 1 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyExpressions: ConstantVectorExpression(val 1) -> 4:long
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: key:int, value:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint
Reducer 2
- Execution mode: vectorized, llap
+ Execution mode: llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ notVectorizedReason: Key expression for GROUPBY operator: Non-column key expressions not supported for MERGEPARTIAL
+ vectorized: false
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -129,14 +175,14 @@ key value grouping__id _c3
2 NULL 1 1
3 NULL 1 2
4 NULL 1 1
-PREHOOK: query: EXPLAIN
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
GROUPING SETS ((), (key))
HAVING GROUPING__ID = 1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
SELECT key, value, GROUPING__ID, count(*)
FROM T1
GROUP BY key, value
@@ -144,6 +190,10 @@ GROUPING SETS ((), (key))
HAVING GROUPING__ID = 1
POSTHOOK: type: QUERY
Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -161,12 +211,26 @@ STAGE PLANS:
TableScan
alias: t1
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
Select Operator
expressions: key (type: int), value (type: int)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1]
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
+ Group By Vectorization:
+ aggregators: VectorUDAFCountStar(*) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0, col 1, ConstantVectorExpression(val 0) -> 2:long
+ native: false
+ projectedOutputColumns: [0]
keys: _col0 (type: int), _col1 (type: int), 0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
@@ -175,28 +239,76 @@ STAGE PLANS:
key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
sort order: +++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No buckets IS true, No TopN IS true, Uniform Hash IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 12 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: key:int, value:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: bigint
Reducer 2
Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY._col0:int, KEY._col1:int, KEY._col2:int, VALUE._col0:bigint
+ partitionColumnCount: 0
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
+ Group By Vectorization:
+ aggregators: VectorUDAFCountMerge(col 3) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0, col 1, col 2
+ native: false
+ projectedOutputColumns: [0]
keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 40 Basic stats: COMPLETE Column stats: NONE
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterLongColEqualLongScalar(col 2, val 1) -> boolean
predicate: (_col2 = 1) (type: boolean)
Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int), 1 (type: int), _col3 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 1, 4, 3]
+ selectExpressions: ConstantVectorExpression(val 1) -> 4:long
Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 3 Data size: 20 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
index b58e707..b4703be 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
@@ -240,3 +240,259 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (1) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (1) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongScalarLongScalar(col 13, val 1, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongScalarLongScalar(col 14, val 1, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0) -> bigint, VectorUDAFSumLong(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+5110 4607
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (cint) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (cint) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongColumnLongScalar(col 13, col 2, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongColumnLongScalar(col 14, col 2, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0) -> bigint, VectorUDAFSumLong(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+248718130534 1995744891643
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
index 67ed3dd..30e17c4 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
@@ -238,3 +238,255 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (1) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (1) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongScalarLongScalar(col 13, val 1, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongScalarLongScalar(col 14, val 1, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0) -> bigint, VectorUDAFSumLong(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+5110 4607
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (cint) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (cint) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongColumnLongScalar(col 13, col 2, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongColumnLongScalar(col 14, col 2, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 0) -> bigint, VectorUDAFSumLong(col 1) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+248718130534 1995744891643
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/vector_between_columns.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_between_columns.q.out b/ql/src/test/results/clientpositive/vector_between_columns.q.out
index cf50a0f..8a79a51 100644
--- a/ql/src/test/results/clientpositive/vector_between_columns.q.out
+++ b/ql/src/test/results/clientpositive/vector_between_columns.q.out
@@ -135,7 +135,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumns: [0, 2, 1, 3, 5]
- selectExpressions: VectorUDFAdaptor(CASE WHEN (_col1 BETWEEN _col3 AND _col3) THEN ('Ok') ELSE ('NoOk') END)(children: VectorUDFAdaptor(_col1 BETWEEN _col3 AND _col3) -> 4:boolean) -> 5:string
+ selectExpressions: IfExprStringScalarStringScalar(col 4, val Ok, val NoOk)(children: VectorUDFAdaptor(_col1 BETWEEN _col3 AND _col3) -> 4:boolean) -> 5:String
Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/cd4fcefb/ql/src/test/results/clientpositive/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_case.q.out b/ql/src/test/results/clientpositive/vectorized_case.q.out
index 20c58ba..2596788 100644
--- a/ql/src/test/results/clientpositive/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_case.q.out
@@ -232,3 +232,225 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (1) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (1) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongScalarLongScalar(col 13, val 1, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongScalarLongScalar(col 14, val 1, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then 1 else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then 1 else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+5110 4607
+PREHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Select Operator
+ expressions: CASE WHEN (((cint % 2) = 0)) THEN (cint) ELSE (0) END (type: int), CASE WHEN (((cint % 2) = 1)) THEN (cint) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [12, 13]
+ selectExpressions: IfExprLongColumnLongScalar(col 13, col 2, val 0)(children: LongColEqualLongScalar(col 12, val 0)(children: LongColModuloLongScalar(col 2, val 2) -> 12:long) -> 13:long) -> 12:long, IfExprLongColumnLongScalar(col 14, col 2, val 0)(children: LongColEqualLongScalar(col 13, val 1)(children: LongColModuloLongScalar(col 2, val 2) -> 13:long) -> 14:long) -> 13:long
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFSumLong(col 12) -> bigint, VectorUDAFSumLong(col 13) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ native: false
+ projectedOutputColumns: [0, 1]
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No buckets IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, Uniform Hash IS false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ sum(case when cint % 2 = 0 then cint else 0 end) as ceven,
+ sum(case when cint % 2 = 1 then cint else 0 end) as codd
+from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+248718130534 1995744891643