You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/06/20 23:16:16 UTC
hive git commit: HIVE-16731: Vectorization: Make "CASE WHEN
(day_name='Sunday') THEN column1 ELSE null end" that involves a column name
or expression THEN or ELSE vectorize (Teddy Choi, reviwed by Matt McCline)
Repository: hive
Updated Branches:
refs/heads/master 8c5f55ee3 -> 4d141c1cd
HIVE-16731: Vectorization: Make "CASE WHEN (day_name='Sunday') THEN column1 ELSE null end" that involves a column name or expression THEN or ELSE vectorize (Teddy Choi, reviwed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4d141c1c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4d141c1c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4d141c1c
Branch: refs/heads/master
Commit: 4d141c1cd5d53a4aad58364ca36c01da058d7665
Parents: 8c5f55e
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Tue Jun 20 18:16:00 2017 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Jun 20 18:16:00 2017 -0500
----------------------------------------------------------------------
.../ql/exec/vector/VectorizationContext.java | 77 +++++++------
.../vector/expressions/IfExprColumnNull.java | 109 +++++++++++++++++++
.../vector/expressions/IfExprNullColumn.java | 109 +++++++++++++++++++
.../llap/vector_adaptor_usage_mode.q.out | 35 ++++--
.../vector_groupby_grouping_sets_grouping.q.out | 2 +-
.../llap/vector_ptf_part_simple.q.out | 18 +--
.../llap/vector_when_case_null.q.out | 8 +-
.../clientpositive/llap/vectorized_case.q.out | 12 +-
.../clientpositive/spark/vectorized_case.q.out | 12 +-
.../clientpositive/vector_when_case_null.q.out | 8 +-
.../clientpositive/vectorized_case.q.out | 12 +-
11 files changed, 324 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index c3940cb..7dc4c81 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -94,7 +94,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -102,7 +101,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.udf.*;
import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
@@ -126,8 +124,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hive.common.util.DateUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -2322,13 +2318,10 @@ public class VectorizationContext {
return createVectorExpression(cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
- private boolean isColumnOrNonNullConst(ExprNodeDesc exprNodeDesc) {
- if (exprNodeDesc instanceof ExprNodeColumnDesc) {
- return true;
- }
+ private boolean isNullConst(ExprNodeDesc exprNodeDesc) {
if (exprNodeDesc instanceof ExprNodeConstantDesc) {
String typeString = exprNodeDesc.getTypeString();
- if (!typeString.equalsIgnoreCase("void")) {
+ if (typeString.equalsIgnoreCase("void")) {
return true;
}
}
@@ -2341,33 +2334,47 @@ public class VectorizationContext {
if (mode != VectorExpressionDescriptor.Mode.PROJECTION) {
return null;
}
- if (childExpr.size() != 3) {
- // For now, we only optimize the 2 value case.
- return null;
- }
+ final int size = childExpr.size();
- /*
- * When we have 2 simple values:
- * CASE WHEN boolExpr THEN column | const ELSE column | const END
- * then we can convert to: IF (boolExpr THEN column | const ELSE column | const)
- */
- // CONSIDER: Adding a version of IfExpr* than can handle a non-column/const expression in the
- // THEN or ELSE.
- ExprNodeDesc exprNodeDesc1 = childExpr.get(1);
- ExprNodeDesc exprNodeDesc2 = childExpr.get(2);
- if (isColumnOrNonNullConst(exprNodeDesc1) &&
- isColumnOrNonNullConst(exprNodeDesc2)) {
- // Yes.
- GenericUDFIf genericUDFIf = new GenericUDFIf();
- return
- getVectorExpressionForUdf(
- genericUDFIf,
- GenericUDFIf.class,
- childExpr,
- mode,
- returnType);
- }
- return null; // Not handled by vector classes yet.
+ final ExprNodeDesc whenDesc = childExpr.get(0);
+ final ExprNodeDesc thenDesc = childExpr.get(1);
+ final ExprNodeDesc elseDesc;
+
+ if (size == 2) {
+ elseDesc = new ExprNodeConstantDesc(returnType, null);
+ } else if (size == 3) {
+ elseDesc = childExpr.get(2);
+ } else {
+ final GenericUDFWhen udfWhen = new GenericUDFWhen();
+ elseDesc = new ExprNodeGenericFuncDesc(returnType, udfWhen, udfWhen.getUdfName(),
+ childExpr.subList(2, childExpr.size()));
+ }
+
+ if (isNullConst(thenDesc)) {
+ final VectorExpression whenExpr = getVectorExpression(whenDesc, mode);
+ final VectorExpression elseExpr = getVectorExpression(elseDesc, mode);
+ final VectorExpression resultExpr = new IfExprNullColumn(
+ whenExpr.getOutputColumn(), elseExpr.getOutputColumn(),
+ ocm.allocateOutputColumn(returnType));
+ resultExpr.setChildExpressions(new VectorExpression[] {whenExpr, elseExpr});
+ resultExpr.setOutputType(returnType.getTypeName());
+ return resultExpr;
+ }
+ if (isNullConst(elseDesc)) {
+ final VectorExpression whenExpr = getVectorExpression(whenDesc, mode);
+ final VectorExpression thenExpr = getVectorExpression(thenDesc, mode);
+ final VectorExpression resultExpr = new IfExprColumnNull(
+ whenExpr.getOutputColumn(), thenExpr.getOutputColumn(),
+ ocm.allocateOutputColumn(returnType));
+ resultExpr.setChildExpressions(new VectorExpression[] {whenExpr, thenExpr});
+ resultExpr.setOutputType(returnType.getTypeName());
+ return resultExpr;
+ }
+ final GenericUDFIf genericUDFIf = new GenericUDFIf();
+ final List<ExprNodeDesc> ifChildExpr = Arrays.<ExprNodeDesc>asList(whenDesc, thenDesc, elseDesc);
+ final ExprNodeGenericFuncDesc exprNodeDesc =
+ new ExprNodeGenericFuncDesc(returnType, genericUDFIf, "if", ifChildExpr);
+ return getVectorExpression(exprNodeDesc, mode);
}
/*
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java
new file mode 100644
index 0000000..8cae274
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+public class IfExprColumnNull extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int arg1Column;
+ private final int arg2Column;
+ private final int outputColumn;
+
+ public IfExprColumnNull(int arg1Column, int arg2Column, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ final LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ final ColumnVector arg2ColVector = batch.cols[arg2Column];
+ final ColumnVector outputColVector = batch.cols[outputColumn];
+
+ final int[] sel = batch.selected;
+ final int n = batch.size;
+ final boolean[] null1 = arg1ColVector.isNull;
+ final long[] vector1 = arg1ColVector.vector;
+ final boolean[] isNull = outputColVector.isNull;
+
+ if (n == 0) {
+ return;
+ }
+
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.isRepeating) {
+ if (!null1[0] && vector1[0] == 1) {
+ outputColVector.setElement(0, 0, arg2ColVector);
+ } else {
+ outputColVector.noNulls = false;
+ isNull[0] = true;
+ }
+ return;
+ }
+ if (batch.selectedInUse) {
+ for (int j = 0; j < n; j++) {
+ int i = sel[j];
+ if (!null1[0] && vector1[i] == 1) {
+ outputColVector.setElement(i, i, arg2ColVector);
+ } else {
+ outputColVector.noNulls = false;
+ isNull[i] = true;
+ }
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ if (!null1[0] && vector1[i] == 1) {
+ outputColVector.setElement(i, i, arg2ColVector);
+ } else {
+ outputColVector.noNulls = false;
+ isNull[i] = true;
+ }
+ }
+ }
+
+ arg2ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return "col " + arg1Column + ", col "+ arg2Column + ", null";
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ throw new UnsupportedOperationException("Undefined descriptor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java
new file mode 100644
index 0000000..156fcc4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+public class IfExprNullColumn extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int arg1Column;
+ private final int arg2Column;
+ private final int outputColumn;
+
+ public IfExprNullColumn(int arg1Column, int arg2Column, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ final LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ final ColumnVector arg2ColVector = batch.cols[arg2Column];
+ final ColumnVector outputColVector = batch.cols[outputColumn];
+
+ final int[] sel = batch.selected;
+ final int n = batch.size;
+ final boolean[] null1 = arg1ColVector.isNull;
+ final long[] vector1 = arg1ColVector.vector;
+ final boolean[] isNull = outputColVector.isNull;
+
+ if (n == 0) {
+ return;
+ }
+
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.isRepeating) {
+ if (!null1[0] && vector1[0] == 1) {
+ outputColVector.noNulls = false;
+ isNull[0] = true;
+ } else {
+ outputColVector.setElement(0, 0, arg2ColVector);
+ }
+ return;
+ }
+ if (batch.selectedInUse) {
+ for (int j = 0; j < n; j++) {
+ int i = sel[j];
+ if (!null1[0] && vector1[i] == 1) {
+ outputColVector.noNulls = false;
+ isNull[i] = true;
+ } else {
+ outputColVector.setElement(i, i, arg2ColVector);
+ }
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ if (!null1[0] && vector1[i] == 1) {
+ outputColVector.noNulls = false;
+ isNull[i] = true;
+ } else {
+ outputColVector.setElement(i, i, arg2ColVector);
+ }
+ }
+ }
+
+ arg2ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return "col " + arg1Column + ", null, col "+ arg2Column;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ throw new UnsupportedOperationException("Undefined descriptor");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
index a2ce365..6a0f490 100644
--- a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out
@@ -965,12 +965,27 @@ STAGE PLANS:
TableScan
alias: count_case_groupby
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1]
Select Operator
expressions: key (type: string), CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END (type: int)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [0, 5]
+ selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1)
+ Group By Vectorization:
+ aggregators: VectorUDAFCount(col 5) -> bigint
+ className: VectorGroupByOperator
+ vectorOutput: true
+ keyExpressions: col 0
+ native: false
+ projectedOutputColumns: [0]
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
@@ -979,16 +994,22 @@ STAGE PLANS:
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: Select expression for SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFWhen(Column[bool], Const int 1, GenericUDFOPNot(Column[bool]), Const int 0, Const void null) because hive.vectorized.adaptor.usage.mode=none
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
Reducer 2
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -1078,13 +1099,13 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 3]
- selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int
+ projectedOutputColumns: [0, 5]
+ selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 3) -> bigint
+ aggregators: VectorUDAFCount(col 5) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 0
@@ -1112,7 +1133,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
Execution mode: vectorized, llap
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out
index 98cd4d1..fccb6d6 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out
@@ -772,7 +772,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
- Execution mode: vectorized, llap
+ Execution mode: llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
index c9c7d8c..c2f5a29 100644
--- a/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out
@@ -2773,8 +2773,8 @@ STAGE PLANS:
Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp)
Reduce Sink Vectorization:
className: VectorReduceSinkMultiKeyOperator
- keyColumns: [0, 4]
- keyExpressions: VectorUDFAdaptor(CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean) -> 4:timestamp
+ keyColumns: [0, 5]
+ keyExpressions: IfExprColumnNull(col 3, col 4, null)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean, ConstantVectorExpression(val 2000-01-01 00:00:00.0) -> 4:timestamp) -> 5:timestamp
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
valueColumns: [1, 2]
@@ -2788,14 +2788,14 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
rowBatchContext:
dataColumnCount: 3
includeColumns: [0, 1, 2]
dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
partitionColumnCount: 0
- scratchColumnTypeNames: bigint, timestamp
+ scratchColumnTypeNames: bigint, timestamp, timestamp
Reducer 2
Execution mode: llap
Reduce Vectorization:
@@ -2941,11 +2941,11 @@ STAGE PLANS:
Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp)
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
- keyColumns: [0, 4, 1]
- keyExpressions: VectorUDFAdaptor(CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean) -> 4:timestamp
+ keyColumns: [0, 5, 1]
+ keyExpressions: IfExprColumnNull(col 3, col 4, null)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean, ConstantVectorExpression(val 2000-01-01 00:00:00.0) -> 4:timestamp) -> 5:timestamp
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumns: [0, 5]
+ partitionColumns: [0, 8]
valueColumns: [2]
Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE
value expressions: p_retailprice (type: double)
@@ -2957,14 +2957,14 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: true
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
rowBatchContext:
dataColumnCount: 3
includeColumns: [0, 1, 2]
dataColumns: p_mfgr:string, p_name:string, p_retailprice:double
partitionColumnCount: 0
- scratchColumnTypeNames: bigint, timestamp, timestamp
+ scratchColumnTypeNames: bigint, timestamp, timestamp, bigint, timestamp, timestamp
Reducer 2
Execution mode: llap
Reduce Vectorization:
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out b/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out
index 06dde80..28edb6f 100644
--- a/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out
@@ -50,13 +50,13 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 3]
- selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int
+ projectedOutputColumns: [0, 5]
+ selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 3) -> bigint
+ aggregators: VectorUDAFCount(col 5) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 0
@@ -84,7 +84,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Reducer 2
Execution mode: vectorized, llap
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
index beda526..0beaba8 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out
@@ -68,8 +68,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 15, 16]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String
Statistics: Num rows: 6 Data size: 2228 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
@@ -89,7 +89,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0
@@ -210,8 +210,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 16, 19]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String
Statistics: Num rows: 6 Data size: 2228 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
@@ -231,7 +231,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
index 133842d..ffac02c 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
@@ -68,8 +68,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 15, 16]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -88,7 +88,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0
@@ -209,8 +209,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 16, 19]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -229,7 +229,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/vector_when_case_null.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_when_case_null.q.out b/ql/src/test/results/clientpositive/vector_when_case_null.q.out
index a7ab4ef..5ae4b99 100644
--- a/ql/src/test/results/clientpositive/vector_when_case_null.q.out
+++ b/ql/src/test/results/clientpositive/vector_when_case_null.q.out
@@ -44,13 +44,13 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [0, 3]
- selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int
+ projectedOutputColumns: [0, 5]
+ selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long
Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 3) -> bigint
+ aggregators: VectorUDAFCount(col 5) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 0
@@ -78,7 +78,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Reduce Vectorization:
enabled: false
http://git-wip-us.apache.org/repos/asf/hive/blob/4d141c1c/ql/src/test/results/clientpositive/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_case.q.out b/ql/src/test/results/clientpositive/vectorized_case.q.out
index 5a7a8a2..b1b5e54 100644
--- a/ql/src/test/results/clientpositive/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_case.q.out
@@ -65,8 +65,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 15, 16]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -85,7 +85,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0
@@ -203,8 +203,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [1, 14, 15]
- selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string
+ projectedOutputColumns: [1, 16, 19]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -223,7 +223,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
Stage: Stage-0