You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ji...@apache.org on 2014/03/29 00:40:27 UTC
svn commit: r1582915 - in /hive/trunk: ant/src/org/apache/hadoop/hive/ant/
ql/src/gen/vectorization/ExpressionTemplates/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/
ql/src/java/org/a...
Author: jitendra
Date: Fri Mar 28 23:40:26 2014
New Revision: 1582915
URL: http://svn.apache.org/r1582915
Log:
HIVE-6752: Vectorized Between and IN expressions don't work with decimal, date types. (reviewed by Eric Hanson, Sergey)
Added:
hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToString.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDecimalInExpr.java
hive/trunk/ql/src/test/queries/clientpositive/vector_between_in.q
hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out
Modified:
hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1582915&r1=1582914&r2=1582915&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Fri Mar 28 23:40:26 2014
@@ -329,6 +329,9 @@ public class GenVectorCode extends Task
{"FilterColumnBetween", "long", "!"},
{"FilterColumnBetween", "double", "!"},
+ {"FilterDecimalColumnBetween", ""},
+ {"FilterDecimalColumnBetween", "!"},
+
{"ColumnCompareColumn", "Equal", "long", "double", "=="},
{"ColumnCompareColumn", "Equal", "double", "double", "=="},
{"ColumnCompareColumn", "NotEqual", "long", "double", "!="},
@@ -659,6 +662,8 @@ public class GenVectorCode extends Task
generateFilterStringColumnCompareScalar(tdesc);
} else if (tdesc[0].equals("FilterStringColumnBetween")) {
generateFilterStringColumnBetween(tdesc);
+ } else if (tdesc[0].equals("FilterDecimalColumnBetween")) {
+ generateFilterDecimalColumnBetween(tdesc);
} else if (tdesc[0].equals("StringColumnCompareScalar")) {
generateStringColumnCompareScalar(tdesc);
} else if (tdesc[0].equals("FilterStringScalarCompareColumn")) {
@@ -705,6 +710,20 @@ public class GenVectorCode extends Task
className, templateString);
}
+ private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOException {
+ String optionalNot = tdesc[1];
+ String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "")
+ + "Between";
+ // Read the template into a string, expand it, and write it.
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<OptionalNot>", optionalNot);
+
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
private void generateFilterColumnBetween(String[] tdesc) throws Exception {
String operandType = tdesc[1];
String optionalNot = tdesc[2];
Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt Fri Mar 28 23:40:26 2014
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Generated from template FilterDecimalColumnBetween.txt, which covers [NOT] BETWEEN filter
+ * expressions where a column is [NOT] between one scalar and another.
+ * Output is not produced in a separate column. The selected vector of the input
+ * {@link VectorizedRowBatch} is updated for in-place filtering.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int colNum;
+
+ // The comparison is of the form "column BETWEEN leftValue AND rightValue"
+ private Decimal128 leftValue;
+ private Decimal128 rightValue;
+
+ public <ClassName>(int colNum, Decimal128 leftValue, Decimal128 rightValue) {
+ this.colNum = colNum;
+ this.leftValue = leftValue;
+ this.rightValue = rightValue;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum];
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector.isNull;
+ int n = batch.size;
+ Decimal128[] vector = inputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero.
+ // Repeating property will not change.
+ if (<OptionalNot>(vector[0].compareTo(leftValue) < 0 || vector[0].compareTo(rightValue) > 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (<OptionalNot>(leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0)) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (<OptionalNot>(leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0)) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero.
+ // Repeating property will not change.
+ if (!nullPos[0]) {
+ if (<OptionalNot>(vector[0].compareTo(leftValue) < 0 || vector[0].compareTo(rightValue) > 0)) {
+
+ // Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (<OptionalNot>(leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0)) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+
+ // Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (<OptionalNot>(leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0)) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ public int getColNum() {
+ return colNum;
+ }
+
+ public void setColNum(int colNum) {
+ this.colNum = colNum;
+ }
+
+ public Decimal128 getLeftValue() {
+ return leftValue;
+ }
+
+ public void setLeftValue(Decimal128 value) {
+ this.leftValue = value;
+ }
+
+ public Decimal128 getRightValue() {
+ return rightValue;
+ }
+
+ public void setRightValue(Decimal128 value) {
+ this.leftValue = value;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.FILTER)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+ VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java?rev=1582915&r1=1582914&r2=1582915&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java Fri Mar 28 23:40:26 2014
@@ -222,7 +222,8 @@ public class VectorHashKeyWrapper extend
}
public void assignDecimal(int index, Decimal128 value) {
- decimalValues[index].update(value);
+ decimalValues[index].update(value);
+ isNull[longValues.length + doubleValues.length + byteValues.length + index] = false;
}
public void assignNullDecimal(int index) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1582915&r1=1582914&r2=1582915&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Fri Mar 28 23:40:26 2014
@@ -41,31 +41,7 @@ import org.apache.hadoop.hive.ql.exec.Fu
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastBooleanToStringViaLongToString;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToDouble;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToString;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDoubleToDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastLongToString;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.DoubleColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterConstantBooleanVectorExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterDoubleColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterLongColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.IDoubleInExpr;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.ILongInExpr;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.IStringInExpr;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsTrue;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.StringColumnInList;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLength;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorCoalesce;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount;
@@ -95,15 +71,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFVarSampDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFVarSampDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFVarSampLong;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToBooleanViaLongToLong;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToDouble;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastTimestampToDoubleViaLongToDouble;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -124,27 +92,7 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFToLong;
import org.apache.hadoop.hive.ql.udf.UDFToShort;
import org.apache.hadoop.hive.ql.udf.UDFToString;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNegative;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPositive;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFRound;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -404,8 +352,11 @@ public class VectorizationContext {
if (commonType == null) {
commonType = returnType;
}
- } else {
+ } else if (genericUdf instanceof GenericUDFIn) {
+ // Cast to the type of the first child
+ return children.get(0).getTypeInfo();
+ } else {
// The children type should be converted to return type
commonType = returnType;
}
@@ -433,6 +384,13 @@ public class VectorizationContext {
}
TypeInfo commonType = getCommonTypeForChildExpressions(genericUDF, children, returnType);
+
+ if (commonType == null) {
+
+ // Couldn't determine common type, don't cast
+ return children;
+ }
+
List<ExprNodeDesc> childrenWithCasts = new ArrayList<ExprNodeDesc>();
boolean atleastOneCastNeeded = false;
for (ExprNodeDesc child : children) {
@@ -453,7 +411,7 @@ public class VectorizationContext {
private boolean isExcludedFromCast(GenericUDF genericUDF) {
boolean ret = castExpressionUdfs.contains(genericUDF.getClass())
- || (genericUDF instanceof GenericUDFRound);
+ || (genericUDF instanceof GenericUDFRound) || (genericUDF instanceof GenericUDFBetween);
if (ret) {
return ret;
@@ -510,9 +468,7 @@ public class VectorizationContext {
// If castType is decimal, try not to lose precision for numeric types.
castType = updatePrecision(inputTypeInfo, (DecimalTypeInfo) castType);
GenericUDFToDecimal castToDecimalUDF = new GenericUDFToDecimal();
- castToDecimalUDF.setTypeInfo(new DecimalTypeInfo(
- getPrecisionForType((PrimitiveTypeInfo) castType),
- HiveDecimalUtils.getScaleForType((PrimitiveTypeInfo) castType)));
+ castToDecimalUDF.setTypeInfo(castType);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
children.add(child);
ExprNodeDesc desc = new ExprNodeGenericFuncDesc(castType, castToDecimalUDF, children);
@@ -689,13 +645,27 @@ public class VectorizationContext {
return exprDesc;
}
- if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ||
- (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc))) {
+ if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
return exprDesc;
}
- ExprNodeConstantDesc encd = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
- ObjectInspector childoi = encd.getWritableObjectInspector();
+ ExprNodeConstantDesc foldedChild = null;
+ if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
+
+ // try recursive folding
+ ExprNodeDesc expr = foldConstantsForUnaryExpression(exprDesc.getChildren().get(0));
+ if (expr instanceof ExprNodeConstantDesc) {
+ foldedChild = (ExprNodeConstantDesc) expr;
+ }
+ } else {
+ foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
+ }
+
+ if (foldedChild == null) {
+ return exprDesc;
+ }
+
+ ObjectInspector childoi = foldedChild.getWritableObjectInspector();
GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive
@@ -917,9 +887,9 @@ public class VectorizationContext {
childExpr = constantFoldedChildren;
//First handle special cases
if (udf instanceof GenericUDFBetween) {
- return getBetweenFilterExpression(childExpr, mode);
+ return getBetweenFilterExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFIn) {
- return getInExpression(childExpr, mode);
+ return getInExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFOPPositive) {
return getIdentityExpression(childExpr);
} else if (udf instanceof GenericUDFCoalesce) {
@@ -981,12 +951,13 @@ public class VectorizationContext {
/**
* Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
- private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, Mode mode)
+ private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType)
throws HiveException {
ExprNodeDesc colExpr = childExpr.get(0);
+
TypeInfo colTypeInfo = colExpr.getTypeInfo();
String colType = colExpr.getTypeString();
-
+
// prepare arguments for createVectorExpression
List<ExprNodeDesc> childrenForInList =
foldConstantsForUnaryExprs(childExpr.subList(1, childExpr.size()));
@@ -1009,23 +980,23 @@ public class VectorizationContext {
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, colTypeInfo);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
- } else if (colType.equals("timestamp")) {
+ } else if (isTimestampFamily(colType)) {
cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getTimestampScalar(childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, colTypeInfo);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
- } else if (colType.equals("string")) {
+ } else if (isStringFamily(colType)) {
cl = (mode == Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);
byte[][] inVals = new byte[childrenForInList.size()][];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, colTypeInfo);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
((IStringInExpr) expr).setInListValues(inVals);
} else if (isFloatFamily(colType)) {
cl = (mode == Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);
@@ -1033,10 +1004,27 @@ public class VectorizationContext {
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, colTypeInfo);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
((IDoubleInExpr) expr).setInListValues(inValsD);
- }
-
+ } else if (isDecimalFamily(colType)) {
+ cl = (mode == Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class);
+ Decimal128[] inValsD = new Decimal128[childrenForInList.size()];
+ for (int i = 0; i != inValsD.length; i++) {
+ inValsD[i] = (Decimal128) getVectorTypeScalarValue(
+ (ExprNodeConstantDesc) childrenForInList.get(i));
+ }
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ ((IDecimalInExpr) expr).setInListValues(inValsD);
+ } else if (isDateFamily(colType)) {
+ cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
+ long[] inVals = new long[childrenForInList.size()];
+ for (int i = 0; i != inVals.length; i++) {
+ inVals[i] = (Integer) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i));
+ }
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ ((ILongInExpr) expr).setInListValues(inVals);
+ }
+
// Return the desired VectorExpression if found. Otherwise, return null to cause
// execution to fall back to row mode.
return expr;
@@ -1145,6 +1133,8 @@ public class VectorizationContext {
return createVectorExpression(CastLongToString.class, childExpr, Mode.PROJECTION, null);
} else if (isDecimalFamily(inputType)) {
return createVectorExpression(CastDecimalToString.class, childExpr, Mode.PROJECTION, returnType);
+ } else if (isDateFamily(inputType)) {
+ return createVectorExpression(CastDateToString.class, childExpr, Mode.PROJECTION, returnType);
}
/* The string type is deliberately omitted -- the planner removes string to string casts.
* Timestamp, float, and double types are handled by the legacy code path. See isLegacyPathUDF.
@@ -1212,7 +1202,7 @@ public class VectorizationContext {
* needs to be done differently than the standard way where all arguments are
* passed to the VectorExpression constructor.
*/
- private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr, Mode mode)
+ private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType)
throws HiveException {
if (mode == Mode.PROJECTION) {
@@ -1225,14 +1215,39 @@ public class VectorizationContext {
boolean notKeywordPresent = (Boolean) ((ExprNodeConstantDesc) childExpr.get(0)).getValue();
ExprNodeDesc colExpr = childExpr.get(1);
- // To hold left and right boundaries as long value in nanos for timestamp type.
- long left, right;
- List<ExprNodeDesc> newChildren;
+ // The children after not, might need a cast. Get common types for the two comparisons.
+ // Casting for 'between' is handled here as a special case, because the first child is for NOT and doesn't need
+ // cast
+ TypeInfo commonType = FunctionRegistry.getCommonClassForComparison(childExpr.get(1).getTypeInfo(),
+ childExpr.get(2).getTypeInfo());
+ if (commonType == null) {
- String colType = colExpr.getTypeString();
+ // Can't vectorize
+ return null;
+ }
+ commonType = FunctionRegistry.getCommonClassForComparison(commonType, childExpr.get(3).getTypeInfo());
+ if (commonType == null) {
+
+ // Can't vectorize
+ return null;
+ }
+
+ List<ExprNodeDesc> castChildren = new ArrayList<ExprNodeDesc>();
+
+ for (ExprNodeDesc desc: childExpr.subList(1, 4)) {
+ if (commonType.equals(desc.getTypeInfo())) {
+ castChildren.add(desc);
+ } else {
+ GenericUDF castUdf = getGenericUDFForCast(commonType);
+ ExprNodeGenericFuncDesc engfd = new ExprNodeGenericFuncDesc(commonType, castUdf,
+ Arrays.asList(new ExprNodeDesc[] { desc }));
+ castChildren.add(engfd);
+ }
+ }
+ String colType = commonType.getTypeName();
// prepare arguments for createVectorExpression
- List<ExprNodeDesc> childrenAfterNot = foldConstantsForUnaryExprs(childExpr.subList(1, 4));;
+ List<ExprNodeDesc> childrenAfterNot = foldConstantsForUnaryExprs(castChildren);
// determine class
Class<?> cl = null;
@@ -1251,8 +1266,8 @@ public class VectorizationContext {
} else if (colType.equals("timestamp")) {
// Get timestamp boundary values as longs instead of the expected strings
- left = getTimestampScalar(childExpr.get(2));
- right = getTimestampScalar(childExpr.get(3));
+ long left = getTimestampScalar(childExpr.get(2));
+ long right = getTimestampScalar(childExpr.get(3));
childrenAfterNot = new ArrayList<ExprNodeDesc>();
childrenAfterNot.add(colExpr);
childrenAfterNot.add(new ExprNodeConstantDesc(left));
@@ -1262,9 +1277,16 @@ public class VectorizationContext {
} else {
cl = FilterLongColumnBetween.class;
}
- }
-
- return createVectorExpression(cl, childrenAfterNot, Mode.PROJECTION, null);
+ } else if (isDecimalFamily(colType) && !notKeywordPresent) {
+ cl = FilterDecimalColumnBetween.class;
+ } else if (isDecimalFamily(colType) && notKeywordPresent) {
+ cl = FilterDecimalColumnNotBetween.class;
+ } else if (isDateFamily(colType) && !notKeywordPresent) {
+ cl = FilterLongColumnBetween.class;
+ } else if (isDateFamily(colType) && notKeywordPresent) {
+ cl = FilterLongColumnNotBetween.class;
+ }
+ return createVectorExpression(cl, childrenAfterNot, Mode.PROJECTION, returnType);
}
/*
@@ -1407,7 +1429,7 @@ public class VectorizationContext {
} else if (o instanceof Long) {
return (Long) o;
}
- throw new HiveException("Unexpected type when converting to long");
+ throw new HiveException("Unexpected type when converting to long : "+o.getClass().getSimpleName());
}
private double getNumericScalarAsDouble(ExprNodeDesc constDesc)
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToString.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToString.java?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToString.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToString.java Fri Mar 28 23:40:26 2014
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+
+import java.sql.Date;
+
+public class CastDateToString extends LongToStringUnaryUDF {
+ private static final long serialVersionUID = 1L;
+ protected transient Date dt = new Date(0);
+
+ public CastDateToString() {
+ super();
+ }
+
+ public CastDateToString(int inputColumn, int outputColumn) {
+ super(inputColumn, outputColumn);
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, long[] vector, int i) {
+ dt.setTime(DateWritable.daysToMillis((int) vector[i]));
+ byte[] temp = dt.toString().getBytes();
+ outV.setVal(i, temp, 0, temp.length);
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java Fri Mar 28 23:40:26 2014
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import java.util.HashSet;
+
+/**
+ * Output a boolean value indicating if a column is IN a list of constants.
+ */
+public class DecimalColumnInList extends VectorExpression implements IDecimalInExpr {
+ private static final long serialVersionUID = 1L;
+ private int inputCol;
+ private Decimal128[] inListValues;
+ private int outputColumn;
+
+ // The set object containing the IN list.
+ private transient HashSet<Decimal128> inSet;
+
+ public DecimalColumnInList() {
+ super();
+ inSet = null;
+ }
+
+ /**
+ * After construction you must call setInListValues() to add the values to the IN set.
+ */
+ public DecimalColumnInList(int colNum, int outputColumn) {
+ this.inputCol = colNum;
+ this.outputColumn = outputColumn;
+ inSet = null;
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ if (inSet == null) {
+ inSet = new HashSet<Decimal128>(inListValues.length);
+ for (Decimal128 val : inListValues) {
+ inSet.add(val);
+ }
+ }
+
+ DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[inputCol];
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector.isNull;
+ boolean[] outNulls = outputColVector.isNull;
+ int n = batch.size;
+ Decimal128[] vector = inputColVector.vector;
+ long[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.isRepeating = false;
+ outputColVector.noNulls = inputColVector.noNulls;
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero
+ // Repeating property will not change.
+ outputVector[0] = inSet.contains(vector[0]) ? 1 : 0;
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = inSet.contains(vector[i]) ? 1 : 0;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = inSet.contains(vector[i]) ? 1 : 0;
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!nullPos[0]) {
+ outputVector[0] = inSet.contains(vector[0]) ? 1 : 0;
+ outNulls[0] = false;
+ } else {
+ outNulls[0] = true;
+ }
+ outputColVector.isRepeating = true;
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outNulls[i] = nullPos[i];
+ if (!nullPos[i]) {
+ outputVector[i] = inSet.contains(vector[i]) ? 1 : 0;
+ }
+ }
+ } else {
+ System.arraycopy(nullPos, 0, outNulls, 0, n);
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ outputVector[i] = inSet.contains(vector[i]) ? 1 : 0;
+ }
+ }
+ }
+ }
+ }
+
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public Descriptor getDescriptor() {
+
+ // This VectorExpression (IN) is a special case, so don't return a descriptor.
+ return null;
+ }
+
+ public Decimal128[] getInListValues() {
+ return this.inListValues;
+ }
+
+ public void setInListValues(Decimal128[] a) {
+ this.inListValues = a;
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java Fri Mar 28 23:40:26 2014
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import java.util.HashSet;
+
+/**
+ * Evaluate IN filter on a batch for a vector of decimals.
+ */
+public class FilterDecimalColumnInList extends VectorExpression implements IDecimalInExpr {
+ private static final long serialVersionUID = 1L;
+ private int inputCol;
+ private Decimal128[] inListValues;
+
+ // The set object containing the IN list.
+ private transient HashSet<Decimal128> inSet;
+
+ public FilterDecimalColumnInList() {
+ super();
+ inSet = null;
+ }
+
+ /**
+ * After construction you must call setInListValues() to add the values to the IN set.
+ */
+ public FilterDecimalColumnInList(int colNum) {
+ this.inputCol = colNum;
+ inSet = null;
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ if (inSet == null) {
+ inSet = new HashSet<Decimal128>(inListValues.length);
+ for (Decimal128 val : inListValues) {
+ inSet.add(val);
+ }
+ }
+
+ DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[inputCol];
+ int[] sel = batch.selected;
+ boolean[] nullPos = inputColVector.isNull;
+ int n = batch.size;
+ Decimal128[] vector = inputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (inputColVector.noNulls) {
+ if (inputColVector.isRepeating) {
+
+ // All must be selected otherwise size would be zero
+ // Repeating property will not change.
+
+ if (!(inSet.contains(vector[0]))) {
+ //Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (inSet.contains(vector[i])) {
+ sel[newSize++] = i;
+ }
+ }
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (inSet.contains(vector[i])) {
+ sel[newSize++] = i;
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ } else {
+ if (inputColVector.isRepeating) {
+
+ //All must be selected otherwise size would be zero
+ //Repeating property will not change.
+ if (!nullPos[0]) {
+ if (!inSet.contains(vector[0])) {
+
+ //Entire batch is filtered out.
+ batch.size = 0;
+ }
+ } else {
+ batch.size = 0;
+ }
+ } else if (batch.selectedInUse) {
+ int newSize = 0;
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!nullPos[i]) {
+ if (inSet.contains(vector[i])) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+
+ // Change the selected vector
+ batch.size = newSize;
+ } else {
+ int newSize = 0;
+ for(int i = 0; i != n; i++) {
+ if (!nullPos[i]) {
+ if (inSet.contains(vector[i])) {
+ sel[newSize++] = i;
+ }
+ }
+ }
+ if (newSize < n) {
+ batch.size = newSize;
+ batch.selectedInUse = true;
+ }
+ }
+ }
+ }
+
+
+ @Override
+ public String getOutputType() {
+ return "boolean";
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return -1;
+ }
+
+ @Override
+ public Descriptor getDescriptor() {
+
+ // This VectorExpression (IN) is a special case, so don't return a descriptor.
+ return null;
+ }
+
+ public Decimal128[] getInListValues() {
+ return this.inListValues;
+ }
+
+ public void setInListValues(Decimal128[] a) {
+ this.inListValues = a;
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDecimalInExpr.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDecimalInExpr.java?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDecimalInExpr.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDecimalInExpr.java Fri Mar 28 23:40:26 2014
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+public interface IDecimalInExpr {
+ void setInListValues(Decimal128[] inVals);
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1582915&r1=1582914&r2=1582915&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri Mar 28 23:40:26 2014
@@ -356,7 +356,7 @@ public class Vectorizer implements Physi
Object... nodeOutputs) throws SemanticException {
for (Node n : stack) {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) n;
- if (op.getType().equals(OperatorType.REDUCESINK) &&
+ if ((op.getType().equals(OperatorType.REDUCESINK) || op.getType().equals(OperatorType.FILESINK)) &&
op.getParentOperators().get(0).getType().equals(OperatorType.GROUPBY)) {
return new Boolean(true);
}
@@ -450,7 +450,7 @@ public class Vectorizer implements Physi
assert vContext != null;
- if (op.getType().equals(OperatorType.REDUCESINK) &&
+ if ((op.getType().equals(OperatorType.REDUCESINK) || op.getType().equals(OperatorType.FILESINK)) &&
op.getParentOperators().get(0).getType().equals(OperatorType.GROUPBY)) {
// No need to vectorize
if (!opsDone.contains(op)) {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1582915&r1=1582914&r2=1582915&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Fri Mar 28 23:40:26 2014
@@ -965,12 +965,12 @@ public class TestVectorizationContext {
children1.set(2, new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
children1.set(3, new ExprNodeConstantDesc("2013-11-06 00:00:00.000"));
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
- assertTrue(ve instanceof FilterLongColumnBetween);
+ assertEquals(FilterStringColumnBetween.class, ve.getClass());
// timestamp NOT BETWEEN
children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
- assertTrue(ve instanceof FilterLongColumnNotBetween);
+ assertEquals(FilterStringColumnNotBetween.class, ve.getClass());
}
// Test translation of both IN filters and boolean-valued IN expressions (non-filters).
Added: hive/trunk/ql/src/test/queries/clientpositive/vector_between_in.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_between_in.q?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_between_in.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_between_in.q Fri Mar 28 23:40:26 2014
@@ -0,0 +1,35 @@
+SET hive.vectorized.execution.enabled=true;
+
+CREATE TABLE decimal_date_test STORED AS ORC AS SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, CAST(CAST((CAST(cint AS BIGINT) *ctinyint) AS TIMESTAMP) AS DATE) AS cdate FROM alltypesorc ORDER BY cdate;
+
+EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate;
+
+EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE));
+
+EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1;
+
+EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568);
+
+EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate;
+
+EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate;
+
+EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1;
+
+EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351;
+
+SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate;
+
+SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE));
+
+SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1;
+
+SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568);
+
+SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate;
+
+SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate;
+
+SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1;
+
+SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351;
Added: hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out?rev=1582915&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out Fri Mar 28 23:40:26 2014
@@ -0,0 +1,660 @@
+PREHOOK: query: CREATE TABLE decimal_date_test STORED AS ORC AS SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, CAST(CAST((CAST(cint AS BIGINT) *ctinyint) AS TIMESTAMP) AS DATE) AS cdate FROM alltypesorc ORDER BY cdate
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+POSTHOOK: query: CREATE TABLE decimal_date_test STORED AS ORC AS SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2, CAST(CAST((CAST(cint AS BIGINT) *ctinyint) AS TIMESTAMP) AS DATE) AS cdate FROM alltypesorc ORDER BY cdate
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@decimal_date_test
+PREHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cdate) IN (CAST( '1969-10-26' AS DATE), CAST( '1969-07-14' AS DATE)) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdate (type: date)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: date)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE))
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (not (cdate) IN (CAST( '1969-10-26' AS DATE), CAST( '1969-07-14' AS DATE), CAST( '1970-01-21' AS DATE))) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cdecimal1) IN (2365.8945945946, 881.0135135135, (- 3367.6517567568)) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdecimal1 (type: decimal(20,10))
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: decimal(20,10))
+ sort order: +
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: decimal(20,10))
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (not (cdecimal1) IN (2365.8945945946, 881.0135135135, (- 3367.6517567568))) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cdate BETWEEN CAST( '1969-12-30' AS DATE) AND CAST( '1970-01-02' AS DATE) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdate (type: date)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: date)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cdate NOT BETWEEN CAST( '1968-05-01' AS DATE) AND CAST( '1971-09-01' AS DATE) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdate (type: date)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: date)
+ sort order: +
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: date)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cdecimal1 BETWEEN (- 20) AND 45.9918918919 (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cdecimal1 (type: decimal(20,10))
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: decimal(20,10))
+ sort order: +
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: decimal(20,10))
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: decimal_date_test
+ Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cdecimal1 NOT BETWEEN (- 2000) AND 4390.1351351351 (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE)) ORDER BY cdate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-07-14
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+1969-10-26
+PREHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdate NOT IN (CAST("1969-10-26" AS DATE), CAST("1969-07-14" AS DATE), CAST("1970-01-21" AS DATE))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+5996
+PREHOOK: query: SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 IN (2365.8945945946, 881.0135135135, -3367.6517567568) ORDER BY cdecimal1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+-3367.6517567568
+-3367.6517567568
+-3367.6517567568
+-3367.6517567568
+-3367.6517567568
+-3367.6517567568
+-3367.6517567568
+881.0135135135
+2365.8945945946
+PREHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT IN (2365.8945945946, 881.0135135135, -3367.6517567568)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+9165
+PREHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate BETWEEN CAST("1969-12-30" AS DATE) AND CAST("1970-01-02" AS DATE) ORDER BY cdate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-30
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1969-12-31
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-01
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+1970-01-02
+PREHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS DATE) AND CAST("1971-09-01" AS DATE) ORDER BY cdate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+1968-04-07
+1968-04-09
+1968-04-10
+1968-04-14
+1968-04-16
+1968-04-16
+1968-04-19
+1968-04-23
+1968-04-25
+1968-04-26
+1968-04-27
+1968-04-27
+1968-04-27
+1968-04-29
+1968-04-29
+1968-04-29
+1968-04-29
+1968-04-30
+1971-09-02
+1971-09-04
+1971-09-06
+1971-09-06
+1971-09-06
+1971-09-09
+1971-09-09
+1971-09-15
+1971-09-17
+1971-09-18
+1971-09-21
+1971-09-21
+1971-09-21
+1971-09-22
+1971-09-22
+1971-09-25
+PREHOOK: query: SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+-18.5162162162
+-17.3216216216
+-16.7243243243
+-16.127027027
+-15.5297297297
+-10.7513513514
+-9.5567567568
+-8.3621621622
+-5.972972973
+-3.5837837838
+4.1810810811
+4.7783783784
+4.7783783784
+5.3756756757
+5.972972973
+5.972972973
+11.3486486486
+11.3486486486
+11.9459459459
+14.9324324324
+19.1135135135
+20.3081081081
+22.1
+24.4891891892
+33.4486486486
+34.6432432432
+40.0189189189
+42.4081081081
+43.0054054054
+44.2
+44.2
+44.7972972973
+45.9918918919
+PREHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_date_test
+#### A masked pattern was here ####
+6172