You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2017/11/20 02:59:17 UTC
[2/2] hive git commit: HIVE-16756 : Vectorization:
LongColModuloLongColumn throws java.lang.ArithmeticException: / by zero
(Vihang Karajgaonkar, reviewed by Matt McCline)
HIVE-16756 : Vectorization: LongColModuloLongColumn throws java.lang.ArithmeticException: / by zero (Vihang Karajgaonkar, reviewed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/33dc63bb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/33dc63bb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/33dc63bb
Branch: refs/heads/branch-2
Commit: 33dc63bb3b4ae1fa1aa0ab39827d52ba18984e1d
Parents: 4c45169
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Sun Nov 19 18:58:54 2017 -0800
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Sun Nov 19 18:58:54 2017 -0800
----------------------------------------------------------------------
.../ExpressionTemplates/ColumnDivideColumn.txt | 2 +-
.../expressions/LongColModuloLongColumn.java | 209 +++++++++
.../hive/ql/udf/generic/GenericUDFOPMod.java | 1 +
.../exec/vector/TestVectorizationContext.java | 2 +-
.../TestVectorArithmeticExpressions.java | 49 ++
.../queries/clientpositive/vectorization_div0.q | 12 +-
.../llap/vectorization_div0.q.out | 444 ++++++++++++++-----
.../spark/vectorization_div0.q.out | 434 +++++++++++++-----
.../clientpositive/tez/vectorization_div0.q.out | 443 +++++++++++++-----
.../clientpositive/vectorization_div0.q.out | 414 ++++++++++++-----
.../apache/hadoop/hive/tools/GenVectorCode.java | 1 -
11 files changed, 1566 insertions(+), 445 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
index 04b533a..ee2d10c 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
/**
- * Generated from template ColumnArithmeticColumn.txt, which covers binary arithmetic
+ * Generated from template ColumnDivideColumn.txt, which covers division and modulo
* expressions between columns.
*/
public class <ClassName> extends VectorExpression {
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java
new file mode 100644
index 0000000..3766116
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
+import org.apache.hadoop.hive.ql.exec.vector.*;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * This operation is handled as a special case because Hive
+ * long%long division needs special handling to avoid
+ * for divide by zero exception
+ */
+public class LongColModuloLongColumn extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum1;
+ private int colNum2;
+ private int outputColumn;
+
+ public LongColModuloLongColumn(int colNum1, int colNum2, int outputColumn) {
+ this.colNum1 = colNum1;
+ this.colNum2 = colNum2;
+ this.outputColumn = outputColumn;
+ }
+
+ public LongColModuloLongColumn() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1];
+ LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2];
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ long[] vector1 = inputColVector1.vector;
+ long[] vector2 = inputColVector2.vector;
+ long[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating =
+ inputColVector1.isRepeating && inputColVector2.isRepeating
+ || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
+ || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
+
+ // Handle nulls first
+ NullUtil.propagateNullsColCol(
+ inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
+
+ /* Disregard nulls for processing. In other words,
+ * the arithmetic operation is performed even if one or
+ * more inputs are null. This is to improve speed by avoiding
+ * conditional checks in the inner loop.
+ */
+ boolean hasDivBy0 = false;
+ if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+ long denom = vector2[0];
+ hasDivBy0 = hasDivBy0 || (denom == 0);
+ if (denom != 0) {
+ outputVector[0] = vector1[0] % denom;
+ }
+ } else if (inputColVector1.isRepeating) {
+ final long vector1Value = vector1[0];
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ long denom = vector2[i];
+ hasDivBy0 = hasDivBy0 || (denom == 0);
+ if (denom != 0) {
+ outputVector[i] = vector1Value % denom;
+ }
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ hasDivBy0 = hasDivBy0 || (vector2[i] == 0);
+ if (vector2[i] != 0) {
+ outputVector[i] = vector1Value % vector2[i];
+ }
+ }
+ }
+ } else if (inputColVector2.isRepeating) {
+ final long vector2Value = vector2[0];
+ if (vector2Value == 0) {
+ // Denominator is zero, convert the batch to nulls
+ outputColVector.noNulls = false;
+ outputColVector.isRepeating = true;
+ outputColVector.isNull[0] = true;
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = vector1[i] % vector2Value;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = vector1[i] % vector2Value;
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ long denom = vector2[i];
+ hasDivBy0 = hasDivBy0 || (denom == 0);
+ if (denom != 0) {
+ outputVector[i] = vector1[i] % denom;
+ }
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ hasDivBy0 = hasDivBy0 || (vector2[i] == 0);
+ if (vector2[i] != 0) {
+ outputVector[i] = vector1[i] % vector2[i];
+ }
+ }
+ }
+ }
+
+ /* For the case when the output can have null values, follow
+ * the convention that the data values must be 1 for long and
+ * NaN for double. This is to prevent possible later zero-divide errors
+ * in complex arithmetic expressions like col2 % (col1 - 1)
+ * in the case when some col1 entries are null.
+ */
+ if (!hasDivBy0) {
+ NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n);
+ } else {
+ NullUtil.setNullAndDivBy0DataEntriesLong(
+ outputColVector, batch.selectedInUse, sel, n, inputColVector2);
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "long";
+ }
+
+ public int getColNum1() {
+ return colNum1;
+ }
+
+ public void setColNum1(int colNum1) {
+ this.colNum1 = colNum1;
+ }
+
+ public int getColNum2() {
+ return colNum2;
+ }
+
+ public void setColNum2(int colNum2) {
+ this.colNum2 = colNum2;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return "col " + colNum1 + ", col " + + colNum2;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(2)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("long"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java
index 6d3e82e..c85fdc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColModuloLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
index 9fcb392..66ce378 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprTimestampScalarSc
import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringGroupColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNotNull;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNull;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColModuloLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColumnInList;
import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongScalar;
import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongScalar;
@@ -107,7 +108,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncLnDoubleToDoubl
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncRoundDoubleToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongColumn;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColModuloLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColMultiplyLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColSubtractLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColUnaryMinus;
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java
index ea06ea0..5033958 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
import junit.framework.Assert;
import org.apache.hadoop.hive.common.type.HiveDecimal;
@@ -308,6 +309,54 @@ public class TestVectorArithmeticExpressions {
}
@Test
+ public void testLongColModuloLongColumn() {
+ VectorizedRowBatch batch = getVectorizedRowBatch2LongInLongOut();
+ LongColModuloLongColumn expr = new LongColModuloLongColumn(0, 1, 2);
+ batch.cols[0].isNull[1] = true;
+ batch.cols[0].noNulls = false;
+ batch.cols[1].noNulls = false;
+ LongColumnVector out = (LongColumnVector) batch.cols[2];
+
+ // Set so we can verify they are reset by operation
+ out.noNulls = true;
+ out.isRepeating = true;
+
+ expr.evaluate(batch);
+
+ // 0/0 for entry 0 should be set as NULL
+ assertFalse(out.noNulls);
+ assertTrue(out.isNull[0]);
+
+ // verify NULL output in entry 1 is correct
+ assertTrue(out.isNull[1]);
+
+ // check entries beyond first 2
+ for (int i = 2; i != batch.size; i++) {
+ assertTrue(out.vector[i] == 0L);
+ }
+ assertFalse(out.noNulls);
+ assertFalse(out.isRepeating);
+ }
+
+ private VectorizedRowBatch getVectorizedRowBatch2LongInLongOut() {
+ VectorizedRowBatch batch = new VectorizedRowBatch(3);
+ LongColumnVector lcv, lcv2;
+ lcv = new LongColumnVector();
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ lcv.vector[i] = i * 37;
+ }
+ batch.cols[0] = lcv;
+ lcv2 = new LongColumnVector();
+ batch.cols[1] = lcv2;
+ for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ lcv2.vector[i] = i * 37;
+ }
+ batch.cols[2] = new LongColumnVector();
+ batch.size = VectorizedRowBatch.DEFAULT_SIZE;
+ return batch;
+ }
+
+ @Test
public void testDecimalColAddDecimalColumn() {
VectorizedRowBatch b = getVectorizedRowBatch3DecimalCols();
VectorExpression expr = new DecimalColAddDecimalColumn(0, 1, 2);
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/queries/clientpositive/vectorization_div0.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorization_div0.q b/ql/src/test/queries/clientpositive/vectorization_div0.q
index 025d457..d7b6c3c 100644
--- a/ql/src/test/queries/clientpositive/vectorization_div0.q
+++ b/ql/src/test/queries/clientpositive/vectorization_div0.q
@@ -5,8 +5,8 @@ set hive.fetch.task.conversion=none;
-- TODO: add more stuff here after HIVE-5918 is fixed, such as cbigint and constants
explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100;
-select cdouble / 0.0 from alltypesorc limit 100;
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100;
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100;
-- There are no zeros in the table, but there is 988888, so use it as zero
@@ -25,3 +25,11 @@ from alltypesorc where cdouble >= -500 and cdouble < -199 order by s1, s2 limit
select (cdouble + 200.0) as s1, cbigint / (cdouble + 200.0) as s2, (cdouble + 200.0) / (cdouble + 200.0), cbigint / (cdouble + 200.0), 3 / (cdouble + 200.0), 1.2 / (cdouble + 200.0)
from alltypesorc where cdouble >= -500 and cdouble < -199 order by s1, s2 limit 100;
+-- There are no zeros in the table, but there is 1018195815 in cbigint, 528534767 in cint, so using it to do a divide by zero. ctinyint has a zero so can be used directly
+
+explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100;
+
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out
index 2b5e5a8..31be04e 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out
@@ -1,8 +1,8 @@
PREHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -21,31 +21,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 73400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Select Operator
- expressions: (cdouble / 0.0) (type: double)
- outputColumnNames: _col0
+ expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12]
- selectExpressions: DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
- Statistics: Num rows: 12288 Data size: 98304 Basic stats: COMPLETE Column stats: COMPLETE
+ projectedOutputColumns: [13, 14, 15, 12]
+ selectExpressions: DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 2) -> 12:double) -> 13:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 0) -> 12:double) -> 14:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 3) -> 12:double) -> 15:double, DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
+ Statistics: Num rows: 12288 Data size: 393216 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 100
Limit Vectorization:
className: VectorLimitOperator
native: true
- Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -67,114 +67,114 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L)
from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100
@@ -615,3 +615,223 @@ POSTHOOK: Input: default@alltypesorc
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
+PREHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2, val 500000000) -> boolean, FilterDoubleColGreaterDoubleScalar(col 5, val 1.0E9) -> boolean, FilterLongColEqualLongScalar(col 0, val 0) -> boolean) -> boolean
+ predicate: ((cint > 500000000) or (cdouble > 1.0E9) or (ctinyint = 0)) (type: boolean)
+ Statistics: Num rows: 4227 Data size: 75752 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 0, 13, 14, 15, 16, 17, 12]
+ selectExpressions: LongColDivideLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 13:double, LongColDivideLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 14:double, LongColDivideLongColumn(col 0, col 0) -> 15:double, LongColModuloLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 16:long, LongColModuloLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 17:long, LongColModuloLongColumn(col 0, col 0) -> 12:long
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col3 (type: double), _col4 (type: double)
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: No TopN IS false
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 4, 0, 1, 5, 6, 7, 8]
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 100
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+528534767 NULL -50 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 39 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -32 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -7 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -44 NULL NULL 1.0 NULL NULL 0
+NULL 1018195815 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -57 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 46 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -54 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 18 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -59 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 27 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 41 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 10 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 2 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -37 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -1 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -12 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 26 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 9 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -47 NULL NULL 1.0 NULL NULL 0
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
index 81979d6..97c0e40 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
@@ -1,8 +1,8 @@
PREHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -26,13 +26,13 @@ STAGE PLANS:
native: true
projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Select Operator
- expressions: (cdouble / 0.0) (type: double)
- outputColumnNames: _col0
+ expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12]
- selectExpressions: DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
+ projectedOutputColumns: [13, 14, 15, 12]
+ selectExpressions: DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 2) -> 12:double) -> 13:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 0) -> 12:double) -> 14:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 3) -> 12:double) -> 15:double, DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
@@ -66,114 +66,114 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L)
from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100
@@ -610,3 +610,221 @@ POSTHOOK: Input: default@alltypesorc
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
+PREHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (SORT, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2, val 500000000) -> boolean, FilterDoubleColGreaterDoubleScalar(col 5, val 1.0E9) -> boolean, FilterLongColEqualLongScalar(col 0, val 0) -> boolean) -> boolean
+ predicate: ((cint > 500000000) or (cdouble > 1.0E9) or (ctinyint = 0)) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 0, 13, 14, 15, 16, 17, 12]
+ selectExpressions: LongColDivideLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 13:double, LongColDivideLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 14:double, LongColDivideLongColumn(col 0, col 0) -> 15:double, LongColModuloLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 16:long, LongColModuloLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 17:long, LongColModuloLongColumn(col 0, col 0) -> 12:long
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: double), _col4 (type: double)
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: No TopN IS false
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 4, 0, 1, 5, 6, 7, 8]
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 100 Data size: 3000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 100 Data size: 3000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 100
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+528534767 NULL -50 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 39 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -32 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -7 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -44 NULL NULL 1.0 NULL NULL 0
+NULL 1018195815 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -57 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 46 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -54 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 18 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -59 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 27 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 41 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 10 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 2 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -37 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -1 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -12 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 26 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 9 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -47 NULL NULL 1.0 NULL NULL 0
http://git-wip-us.apache.org/repos/asf/hive/blob/33dc63bb/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out
index cb7dc90..960ecb9 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out
@@ -1,8 +1,8 @@
PREHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization expression
-select cdouble / 0.0 from alltypesorc limit 100
+select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
PLAN VECTORIZATION:
enabled: true
@@ -21,31 +21,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 73400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
Select Operator
- expressions: (cdouble / 0.0) (type: double)
- outputColumnNames: _col0
+ expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumns: [12]
- selectExpressions: DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
- Statistics: Num rows: 12288 Data size: 98304 Basic stats: COMPLETE Column stats: COMPLETE
+ projectedOutputColumns: [13, 14, 15, 12]
+ selectExpressions: DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 2) -> 12:double) -> 13:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 0) -> 12:double) -> 14:double, DoubleColDivideDoubleScalar(col 12, val 0.0)(children: CastLongToDouble(col 3) -> 12:double) -> 15:double, DoubleColDivideDoubleScalar(col 5, val 0.0) -> 12:double
+ Statistics: Num rows: 12288 Data size: 393216 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 100
Limit Vectorization:
className: VectorLimitOperator
native: true
- Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -66,114 +66,114 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100
+POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
-NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L)
from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100
@@ -612,3 +612,222 @@ POSTHOOK: Input: default@alltypesorc
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
0.0 NULL NULL NULL NULL NULL
+PREHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization expression
+select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2, val 500000000) -> boolean, FilterDoubleColGreaterDoubleScalar(col 5, val 1.0E9) -> boolean, FilterLongColEqualLongScalar(col 0, val 0) -> boolean) -> boolean
+ predicate: ((cint > 500000000) or (cdouble > 1.0E9) or (ctinyint = 0)) (type: boolean)
+ Statistics: Num rows: 4227 Data size: 75752 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 0, 13, 14, 15, 16, 17, 12]
+ selectExpressions: LongColDivideLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 13:double, LongColDivideLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 14:double, LongColDivideLongColumn(col 0, col 0) -> 15:double, LongColModuloLongColumn(col 2, col 12)(children: LongColSubtractLongScalar(col 2, val 528534767) -> 12:long) -> 16:long, LongColModuloLongColumn(col 3, col 12)(children: LongColSubtractLongScalar(col 3, val 1018195815) -> 12:long) -> 17:long, LongColModuloLongColumn(col 0, col 0) -> 12:long
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col3 (type: double), _col4 (type: double)
+ sort order: ++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: No TopN IS false
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
+ groupByVectorOutput: true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumns: [2, 3, 4, 0, 1, 5, 6, 7, 8]
+ Statistics: Num rows: 4227 Data size: 219576 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 100
+ Limit Vectorization:
+ className: VectorLimitOperator
+ native: true
+ Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 100
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3
+from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+528534767 NULL -50 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 31 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -62 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 39 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -32 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -7 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -11 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -44 NULL NULL 1.0 NULL NULL 0
+NULL 1018195815 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 24 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -57 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 46 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 29 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -56 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -16 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -54 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -23 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 51 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 19 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 18 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -55 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -40 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 61 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -59 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -33 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -30 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -53 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 34 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 27 NULL NULL 1.0 NULL NULL 0
+528534767 NULL NULL NULL NULL NULL NULL NULL NULL
+528534767 NULL -21 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 41 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -28 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -5 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -45 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 10 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -48 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 2 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -37 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -43 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 36 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -1 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -12 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 0 NULL NULL NULL NULL NULL NULL
+528534767 NULL 26 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -22 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 9 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -13 NULL NULL 1.0 NULL NULL 0
+528534767 NULL 38 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -4 NULL NULL 1.0 NULL NULL 0
+528534767 NULL -47 NULL NULL 1.0 NULL NULL 0