You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/10/27 23:42:22 UTC
svn commit: r1634715 [1/7] - in /hive/branches/spark: ./
ant/src/org/apache/hadoop/hive/ant/ hbase-handler/
itests/src/test/resources/ ql/src/gen/vectorization/ExpressionTemplates/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ ql/src/java/org/apac...
Author: xuefu
Date: Mon Oct 27 22:42:21 2014
New Revision: 1634715
URL: http://svn.apache.org/r1634715
Log:
HIVE-8415: Vectorized comparison of timestamp and integer needs to treat integer as seconds since epoch (Matt McCline via Jason Dere)
merged from trunk r1631927
Added:
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/FilterScalarCompareTimestampColumn.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterScalarCompareTimestampColumn.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareScalar.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareScalar.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareTimestampColumn.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/ScalarCompareTimestampColumn.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareScalar.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareScalar.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
hive/branches/spark/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
- copied unchanged from r1631927, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
hive/branches/spark/ql/src/test/results/clientpositive/tez/vectorization_7.q.out
- copied unchanged from r1631927, hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/vectorization_8.q.out
- copied unchanged from r1631927, hive/trunk/ql/src/test/results/clientpositive/tez/vectorization_8.q.out
Modified:
hive/branches/spark/ (props changed)
hive/branches/spark/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
hive/branches/spark/hbase-handler/pom.xml (props changed)
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_13.q
hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_7.q
hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_8.q
hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_short_regress.q
hive/branches/spark/ql/src/test/results/clientpositive/tez/vectorization_13.q.out
hive/branches/spark/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorization_13.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorization_7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorization_8.q.out
hive/branches/spark/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
Propchange: hive/branches/spark/
------------------------------------------------------------------------------
Merged /hive/trunk:r1631927
Modified: hive/branches/spark/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/branches/spark/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Mon Oct 27 22:42:21 2014
@@ -180,6 +180,46 @@ public class GenVectorCode extends Task
{"ScalarCompareColumn", "GreaterEqual", "long", "long", ">="},
{"ScalarCompareColumn", "GreaterEqual", "double", "long", ">="},
+ {"TimestampColumnCompareTimestampScalar", "Equal"},
+ {"TimestampColumnCompareTimestampScalar", "NotEqual"},
+ {"TimestampColumnCompareTimestampScalar", "Less"},
+ {"TimestampColumnCompareTimestampScalar", "LessEqual"},
+ {"TimestampColumnCompareTimestampScalar", "Greater"},
+ {"TimestampColumnCompareTimestampScalar", "GreaterEqual"},
+
+ {"TimestampColumnCompareScalar", "Equal", "long"},
+ {"TimestampColumnCompareScalar", "Equal", "double"},
+ {"TimestampColumnCompareScalar", "NotEqual", "long"},
+ {"TimestampColumnCompareScalar", "NotEqual", "double"},
+ {"TimestampColumnCompareScalar", "Less", "long"},
+ {"TimestampColumnCompareScalar", "Less", "double"},
+ {"TimestampColumnCompareScalar", "LessEqual", "long"},
+ {"TimestampColumnCompareScalar", "LessEqual", "double"},
+ {"TimestampColumnCompareScalar", "Greater", "long"},
+ {"TimestampColumnCompareScalar", "Greater", "double"},
+ {"TimestampColumnCompareScalar", "GreaterEqual", "long"},
+ {"TimestampColumnCompareScalar", "GreaterEqual", "double"},
+
+ {"TimestampScalarCompareTimestampColumn", "Equal"},
+ {"TimestampScalarCompareTimestampColumn", "NotEqual"},
+ {"TimestampScalarCompareTimestampColumn", "Less"},
+ {"TimestampScalarCompareTimestampColumn", "LessEqual"},
+ {"TimestampScalarCompareTimestampColumn", "Greater"},
+ {"TimestampScalarCompareTimestampColumn", "GreaterEqual"},
+
+ {"ScalarCompareTimestampColumn", "Equal", "long"},
+ {"ScalarCompareTimestampColumn", "Equal", "double"},
+ {"ScalarCompareTimestampColumn", "NotEqual", "long"},
+ {"ScalarCompareTimestampColumn", "NotEqual", "double"},
+ {"ScalarCompareTimestampColumn", "Less", "long"},
+ {"ScalarCompareTimestampColumn", "Less", "double"},
+ {"ScalarCompareTimestampColumn", "LessEqual", "long"},
+ {"ScalarCompareTimestampColumn", "LessEqual", "double"},
+ {"ScalarCompareTimestampColumn", "Greater", "long"},
+ {"ScalarCompareTimestampColumn", "Greater", "double"},
+ {"ScalarCompareTimestampColumn", "GreaterEqual", "long"},
+ {"ScalarCompareTimestampColumn", "GreaterEqual", "double"},
+
{"FilterColumnCompareScalar", "Equal", "long", "double", "=="},
{"FilterColumnCompareScalar", "Equal", "double", "double", "=="},
{"FilterColumnCompareScalar", "NotEqual", "long", "double", "!="},
@@ -232,6 +272,46 @@ public class GenVectorCode extends Task
{"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="},
{"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="},
+ {"FilterTimestampColumnCompareTimestampScalar", "Equal"},
+ {"FilterTimestampColumnCompareTimestampScalar", "NotEqual"},
+ {"FilterTimestampColumnCompareTimestampScalar", "Less"},
+ {"FilterTimestampColumnCompareTimestampScalar", "LessEqual"},
+ {"FilterTimestampColumnCompareTimestampScalar", "Greater"},
+ {"FilterTimestampColumnCompareTimestampScalar", "GreaterEqual"},
+
+ {"FilterTimestampColumnCompareScalar", "Equal", "long"},
+ {"FilterTimestampColumnCompareScalar", "Equal", "double"},
+ {"FilterTimestampColumnCompareScalar", "NotEqual", "long"},
+ {"FilterTimestampColumnCompareScalar", "NotEqual", "double"},
+ {"FilterTimestampColumnCompareScalar", "Less", "long"},
+ {"FilterTimestampColumnCompareScalar", "Less", "double"},
+ {"FilterTimestampColumnCompareScalar", "LessEqual", "long"},
+ {"FilterTimestampColumnCompareScalar", "LessEqual", "double"},
+ {"FilterTimestampColumnCompareScalar", "Greater", "long"},
+ {"FilterTimestampColumnCompareScalar", "Greater", "double"},
+ {"FilterTimestampColumnCompareScalar", "GreaterEqual", "long"},
+ {"FilterTimestampColumnCompareScalar", "GreaterEqual", "double"},
+
+ {"FilterTimestampScalarCompareTimestampColumn", "Equal"},
+ {"FilterTimestampScalarCompareTimestampColumn", "NotEqual"},
+ {"FilterTimestampScalarCompareTimestampColumn", "Less"},
+ {"FilterTimestampScalarCompareTimestampColumn", "LessEqual"},
+ {"FilterTimestampScalarCompareTimestampColumn", "Greater"},
+ {"FilterTimestampScalarCompareTimestampColumn", "GreaterEqual"},
+
+ {"FilterScalarCompareTimestampColumn", "Equal", "long"},
+ {"FilterScalarCompareTimestampColumn", "Equal", "double"},
+ {"FilterScalarCompareTimestampColumn", "NotEqual", "long"},
+ {"FilterScalarCompareTimestampColumn", "NotEqual", "double"},
+ {"FilterScalarCompareTimestampColumn", "Less", "long"},
+ {"FilterScalarCompareTimestampColumn", "Less", "double"},
+ {"FilterScalarCompareTimestampColumn", "LessEqual", "long"},
+ {"FilterScalarCompareTimestampColumn", "LessEqual", "double"},
+ {"FilterScalarCompareTimestampColumn", "Greater", "long"},
+ {"FilterScalarCompareTimestampColumn", "Greater", "double"},
+ {"FilterScalarCompareTimestampColumn", "GreaterEqual", "long"},
+ {"FilterScalarCompareTimestampColumn", "GreaterEqual", "double"},
+
{"FilterStringGroupColumnCompareStringGroupScalarBase", "Equal", "=="},
{"FilterStringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="},
{"FilterStringGroupColumnCompareStringGroupScalarBase", "Less", "<"},
@@ -401,26 +481,26 @@ public class GenVectorCode extends Task
{"FilterColumnCompareColumn", "GreaterEqual", "long", "double", ">="},
{"FilterColumnCompareColumn", "GreaterEqual", "double", "double", ">="},
- {"FilterColumnCompareColumn", "Equal", "long", "long", "=="},
- {"FilterColumnCompareColumn", "Equal", "double", "long", "=="},
- {"FilterColumnCompareColumn", "NotEqual", "long", "long", "!="},
- {"FilterColumnCompareColumn", "NotEqual", "double", "long", "!="},
- {"FilterColumnCompareColumn", "Less", "long", "long", "<"},
- {"FilterColumnCompareColumn", "Less", "double", "long", "<"},
- {"FilterColumnCompareColumn", "LessEqual", "long", "long", "<="},
- {"FilterColumnCompareColumn", "LessEqual", "double", "long", "<="},
- {"FilterColumnCompareColumn", "Greater", "long", "long", ">"},
- {"FilterColumnCompareColumn", "Greater", "double", "long", ">"},
- {"FilterColumnCompareColumn", "GreaterEqual", "long", "long", ">="},
- {"FilterColumnCompareColumn", "GreaterEqual", "double", "long", ">="},
+ {"FilterColumnCompareColumn", "Equal", "long", "long", "=="},
+ {"FilterColumnCompareColumn", "Equal", "double", "long", "=="},
+ {"FilterColumnCompareColumn", "NotEqual", "long", "long", "!="},
+ {"FilterColumnCompareColumn", "NotEqual", "double", "long", "!="},
+ {"FilterColumnCompareColumn", "Less", "long", "long", "<"},
+ {"FilterColumnCompareColumn", "Less", "double", "long", "<"},
+ {"FilterColumnCompareColumn", "LessEqual", "long", "long", "<="},
+ {"FilterColumnCompareColumn", "LessEqual", "double", "long", "<="},
+ {"FilterColumnCompareColumn", "Greater", "long", "long", ">"},
+ {"FilterColumnCompareColumn", "Greater", "double", "long", ">"},
+ {"FilterColumnCompareColumn", "GreaterEqual", "long", "long", ">="},
+ {"FilterColumnCompareColumn", "GreaterEqual", "double", "long", ">="},
{"FilterColumnBetween", "long", ""},
{"FilterColumnBetween", "double", ""},
{"FilterColumnBetween", "long", "!"},
{"FilterColumnBetween", "double", "!"},
- {"FilterDecimalColumnBetween", ""},
- {"FilterDecimalColumnBetween", "!"},
+ {"FilterDecimalColumnBetween", ""},
+ {"FilterDecimalColumnBetween", "!"},
{"ColumnCompareColumn", "Equal", "long", "double", "=="},
{"ColumnCompareColumn", "Equal", "double", "double", "=="},
@@ -718,10 +798,26 @@ public class GenVectorCode extends Task
generateColumnCompareScalar(tdesc);
} else if (tdesc[0].equals("ScalarCompareColumn")) {
generateScalarCompareColumn(tdesc);
+ } else if (tdesc[0].equals("TimestampScalarCompareTimestampColumn")) {
+ generateTimestampScalarCompareTimestampColumn(tdesc);
+ } else if (tdesc[0].equals("ScalarCompareTimestampColumn")) {
+ generateScalarCompareTimestampColumn(tdesc);
+ } else if (tdesc[0].equals("TimestampColumnCompareTimestampScalar")) {
+ generateTimestampColumnCompareTimestampScalar(tdesc);
+ } else if (tdesc[0].equals("TimestampColumnCompareScalar")) {
+ generateTimestampColumnCompareScalar(tdesc);
} else if (tdesc[0].equals("FilterColumnCompareScalar")) {
generateFilterColumnCompareScalar(tdesc);
} else if (tdesc[0].equals("FilterScalarCompareColumn")) {
generateFilterScalarCompareColumn(tdesc);
+ } else if (tdesc[0].equals("FilterTimestampColumnCompareTimestampScalar")) {
+ generateFilterTimestampColumnCompareTimestampScalar(tdesc);
+ } else if (tdesc[0].equals("FilterTimestampColumnCompareScalar")) {
+ generateFilterTimestampColumnCompareScalar(tdesc);
+ } else if (tdesc[0].equals("FilterTimestampScalarCompareTimestampColumn")) {
+ generateFilterTimestampScalarCompareTimestampColumn(tdesc);
+ } else if (tdesc[0].equals("FilterScalarCompareTimestampColumn")) {
+ generateFilterScalarCompareTimestampColumn(tdesc);
} else if (tdesc[0].equals("FilterColumnBetween")) {
generateFilterColumnBetween(tdesc);
} else if (tdesc[0].equals("ScalarArithmeticColumn") || tdesc[0].equals("ScalarDivideColumn")) {
@@ -1253,6 +1349,8 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
templateString = templateString.replaceAll("<OperandType>", operandType);
String vectorExprArgType = operandType;
+
+ // Toss in timestamp and date.
if (operandType.equals("long")) {
// Let comparisons occur for DATE and TIMESTAMP, too.
vectorExprArgType = "int_datetime_family";
@@ -1281,10 +1379,12 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+
String vectorExprArgType2 = operandType2;
String vectorExprArgType3 = operandType3;
+
+ // Toss in timestamp and date.
if (operandType2.equals("long") && operandType3.equals("long")) {
- // Let this work occur for DATE and TIMESTAMP, too.
vectorExprArgType2 = "int_datetime_family";
vectorExprArgType3 = "int_datetime_family";
}
@@ -1312,10 +1412,12 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+
String vectorExprArgType2 = operandType2;
String vectorExprArgType3 = operandType3;
+
+ // Toss in timestamp and date.
if (operandType2.equals("long") && operandType3.equals("long")) {
- // Let this work occur for DATE and TIMESTAMP, too.
vectorExprArgType2 = "int_datetime_family";
vectorExprArgType3 = "int_datetime_family";
}
@@ -1342,10 +1444,12 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+
String vectorExprArgType2 = operandType2;
String vectorExprArgType3 = operandType3;
+
+ // Toss in timestamp and date.
if (operandType2.equals("long") && operandType3.equals("long")) {
- // Let this work occur for DATE and TIMESTAMP, too.
vectorExprArgType2 = "int_datetime_family";
vectorExprArgType3 = "int_datetime_family";
}
@@ -1477,6 +1581,9 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
String vectorExprArgType1 = operandType1;
String vectorExprArgType2 = operandType2;
+
+ // For column to column only, we toss in timestamp and date.
+ // But {timestamp|date} and scalar must be handled separately.
if (operandType1.equals("long") && operandType2.equals("long")) {
// Let comparisons occur for DATE and TIMESTAMP, too.
vectorExprArgType1 = "int_datetime_family";
@@ -1502,6 +1609,133 @@ public class GenVectorCode extends Task
}
}
+ private void generateTimestampScalarCompareTimestampColumn(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String className = "TimestampScalar" + operatorName + "TimestampColumn";
+ String baseClassName = "LongScalar" + operatorName + "LongColumn";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateTimestampColumnCompareTimestampScalar(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String className = "TimestampCol" + operatorName + "TimestampScalar";
+ String baseClassName = "LongCol" + operatorName + "LongScalar";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterTimestampColumnCompareTimestampScalar(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String className = "FilterTimestampCol" + operatorName + "TimestampScalar";
+ String baseClassName = "FilterLongCol" + operatorName + "LongScalar";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterTimestampScalarCompareTimestampColumn(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String className = "FilterTimestampScalar" + operatorName + "TimestampColumn";
+ String baseClassName = "FilterLongScalar" + operatorName + "LongColumn";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private String timestampScalarConversion(String operandType) {
+ if (operandType.equals("long")) {
+ return "secondsToNanoseconds";
+ } else if (operandType.equals("double")) {
+ return "doubleToNanoseconds";
+ } else {
+ return "unknown";
+ }
+ }
+
+ private void generateScalarCompareTimestampColumn(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String operandType = tdesc[2];
+ String className = getCamelCaseType(operandType) + "Scalar" + operatorName + "TimestampColumn";
+ String baseClassName = "LongScalar" + operatorName + "LongColumn";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<TimestampScalarConversion>", timestampScalarConversion(operandType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateTimestampColumnCompareScalar(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String operandType = tdesc[2];
+ String className = "TimestampCol" + operatorName + getCamelCaseType(operandType) + "Scalar";
+ String baseClassName = "LongCol" + operatorName + "LongScalar";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<TimestampScalarConversion>", timestampScalarConversion(operandType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterTimestampColumnCompareScalar(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String operandType = tdesc[2];
+ String className = "FilterTimestampCol" + operatorName + getCamelCaseType(operandType) + "Scalar";
+ String baseClassName = "FilterLongCol" + operatorName + "LongScalar";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<TimestampScalarConversion>", timestampScalarConversion(operandType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterScalarCompareTimestampColumn(String[] tdesc) throws Exception {
+ String operatorName = tdesc[1];
+ String operandType = tdesc[2];
+ String className = "Filter" + getCamelCaseType(operandType) + "Scalar" + operatorName + "TimestampColumn";
+ String baseClassName = "FilterLongScalar" + operatorName + "LongColumn";
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ templateString = templateString.replaceAll("<TimestampScalarConversion>", timestampScalarConversion(operandType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+
private void generateColumnArithmeticOperatorColumn(String[] tdesc, String returnType,
String className) throws Exception {
String operandType1 = tdesc[2];
@@ -1555,11 +1789,6 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
String vectorExprArgType1 = operandType1;
String vectorExprArgType2 = operandType2;
- if (operandType1.equals("long") && operandType2.equals("long")) {
- // Let comparisons occur for DATE and TIMESTAMP, too.
- vectorExprArgType1 = "int_datetime_family";
- vectorExprArgType2 = "int_datetime_family";
- }
templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1);
templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
@@ -1633,11 +1862,6 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
String vectorExprArgType1 = operandType1;
String vectorExprArgType2 = operandType2;
- if (operandType1.equals("long") && operandType2.equals("long")) {
- // Let comparisons occur for DATE and TIMESTAMP, too.
- vectorExprArgType1 = "int_datetime_family";
- vectorExprArgType2 = "int_datetime_family";
- }
templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1);
templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
Propchange: hive/branches/spark/hbase-handler/pom.xml
------------------------------------------------------------------------------
Merged /hive/trunk/hbase-handler/pom.xml:r1631927
Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Mon Oct 27 22:42:21 2014
@@ -186,6 +186,8 @@ minitez.query.files.shared=alter_merge_2
vectorization_4.q,\
vectorization_5.q,\
vectorization_6.q,\
+ vectorization_7.q,\
+ vectorization_8.q,\
vectorization_9.q,\
vectorization_decimal_date.q,\
vectorization_div0.q,\
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java Mon Oct 27 22:42:21 2014
@@ -50,4 +50,12 @@ public final class TimestampUtils {
int nanos = t.getNanos();
return (time * 1000000) + (nanos % 1000000);
}
+
+ public static long secondsToNanoseconds(long seconds) {
+ return seconds * 1000000000;
+ }
+
+ public static long doubleToNanoseconds(double d) {
+ return (long) (d * 1000000000);
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Mon Oct 27 22:42:21 2014
@@ -790,6 +790,8 @@ public class VectorizationContext {
switch (vectorArgType) {
case INT_FAMILY:
return new ConstantVectorExpression(outCol, ((Number) constantValue).longValue());
+ case TIMESTAMP:
+ return new ConstantVectorExpression(outCol, TimestampUtils.getTimeNanoSec((Timestamp) constantValue));
case FLOAT_FAMILY:
return new ConstantVectorExpression(outCol, ((Number) constantValue).doubleValue());
case DECIMAL:
@@ -804,7 +806,7 @@ public class VectorizationContext {
case VARCHAR:
return new ConstantVectorExpression(outCol, ((HiveVarchar) constantValue));
default:
- throw new HiveException("Unsupported constant type: " + typeName);
+ throw new HiveException("Unsupported constant type: " + typeName + ", object class " + constantValue.getClass().getSimpleName());
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java Mon Oct 27 22:42:21 2014
@@ -49,7 +49,14 @@ import org.apache.hadoop.hive.serde2.obj
FilterLongScalarEqualLongColumn.class, FilterLongScalarEqualDoubleColumn.class,
FilterDoubleScalarEqualLongColumn.class, FilterDoubleScalarEqualDoubleColumn.class,
FilterDecimalColEqualDecimalColumn.class, FilterDecimalColEqualDecimalScalar.class,
- FilterDecimalScalarEqualDecimalColumn.class})
+ FilterDecimalScalarEqualDecimalColumn.class,
+ TimestampColEqualTimestampScalar.class, TimestampScalarEqualTimestampColumn.class,
+ FilterTimestampColEqualTimestampScalar.class, FilterTimestampScalarEqualTimestampColumn.class,
+ TimestampColEqualLongScalar.class, LongScalarEqualTimestampColumn.class,
+ FilterTimestampColEqualLongScalar.class, FilterLongScalarEqualTimestampColumn.class,
+ TimestampColEqualDoubleScalar.class, DoubleScalarEqualTimestampColumn.class,
+ FilterTimestampColEqualDoubleScalar.class, FilterDoubleScalarEqualTimestampColumn.class
+ })
public class GenericUDFOPEqual extends GenericUDFBaseCompare {
public GenericUDFOPEqual(){
this.opName = "EQUAL";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java Mon Oct 27 22:42:21 2014
@@ -50,7 +50,14 @@ import org.apache.hadoop.io.Text;
FilterLongScalarGreaterEqualLongColumn.class, FilterLongScalarGreaterEqualDoubleColumn.class,
FilterDoubleScalarGreaterEqualLongColumn.class, FilterDoubleScalarGreaterEqualDoubleColumn.class,
FilterDecimalColGreaterEqualDecimalColumn.class, FilterDecimalColGreaterEqualDecimalScalar.class,
- FilterDecimalScalarGreaterEqualDecimalColumn.class})
+ FilterDecimalScalarGreaterEqualDecimalColumn.class,
+ TimestampColGreaterEqualTimestampScalar.class, TimestampScalarGreaterEqualTimestampColumn.class,
+ FilterTimestampColGreaterEqualTimestampScalar.class, FilterTimestampScalarGreaterEqualTimestampColumn.class,
+ TimestampColGreaterEqualLongScalar.class, LongScalarGreaterEqualTimestampColumn.class,
+ FilterTimestampColGreaterEqualLongScalar.class, FilterLongScalarGreaterEqualTimestampColumn.class,
+ TimestampColGreaterEqualDoubleScalar.class, DoubleScalarGreaterEqualTimestampColumn.class,
+ FilterTimestampColGreaterEqualDoubleScalar.class, FilterDoubleScalarGreaterEqualTimestampColumn.class
+ })
public class GenericUDFOPEqualOrGreaterThan extends GenericUDFBaseCompare {
public GenericUDFOPEqualOrGreaterThan(){
this.opName = "EQUAL OR GREATER THAN";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java Mon Oct 27 22:42:21 2014
@@ -50,7 +50,14 @@ import org.apache.hadoop.io.Text;
FilterLongScalarLessEqualLongColumn.class, FilterLongScalarLessEqualDoubleColumn.class,
FilterDoubleScalarLessEqualLongColumn.class, FilterDoubleScalarLessEqualDoubleColumn.class,
FilterDecimalColLessEqualDecimalColumn.class, FilterDecimalColLessEqualDecimalScalar.class,
- FilterDecimalScalarLessEqualDecimalColumn.class})
+ FilterDecimalScalarLessEqualDecimalColumn.class,
+ TimestampColLessEqualTimestampScalar.class, TimestampScalarLessEqualTimestampColumn.class,
+ FilterTimestampColLessEqualTimestampScalar.class, FilterTimestampScalarLessEqualTimestampColumn.class,
+ TimestampColLessEqualLongScalar.class, LongScalarLessEqualTimestampColumn.class,
+ FilterTimestampColLessEqualLongScalar.class, FilterLongScalarLessEqualTimestampColumn.class,
+ TimestampColLessEqualDoubleScalar.class, DoubleScalarLessEqualTimestampColumn.class,
+ FilterTimestampColLessEqualDoubleScalar.class, FilterDoubleScalarLessEqualTimestampColumn.class
+ })
public class GenericUDFOPEqualOrLessThan extends GenericUDFBaseCompare {
public GenericUDFOPEqualOrLessThan(){
this.opName = "EQUAL OR LESS THAN";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java Mon Oct 27 22:42:21 2014
@@ -50,7 +50,14 @@ import org.apache.hadoop.io.Text;
FilterLongScalarGreaterLongColumn.class, FilterLongScalarGreaterDoubleColumn.class,
FilterDoubleScalarGreaterLongColumn.class, FilterDoubleScalarGreaterDoubleColumn.class,
FilterDecimalColGreaterDecimalColumn.class, FilterDecimalColGreaterDecimalScalar.class,
- FilterDecimalScalarGreaterDecimalColumn.class})
+ FilterDecimalScalarGreaterDecimalColumn.class,
+ TimestampColGreaterTimestampScalar.class, TimestampScalarGreaterTimestampColumn.class,
+ FilterTimestampColGreaterTimestampScalar.class, FilterTimestampScalarGreaterTimestampColumn.class,
+ TimestampColGreaterLongScalar.class, LongScalarGreaterTimestampColumn.class,
+ FilterTimestampColGreaterLongScalar.class, FilterLongScalarGreaterTimestampColumn.class,
+ TimestampColGreaterDoubleScalar.class, DoubleScalarGreaterTimestampColumn.class,
+ FilterTimestampColGreaterDoubleScalar.class, FilterDoubleScalarGreaterTimestampColumn.class
+ })
public class GenericUDFOPGreaterThan extends GenericUDFBaseCompare {
public GenericUDFOPGreaterThan(){
this.opName = "GREATER THAN";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java Mon Oct 27 22:42:21 2014
@@ -50,7 +50,14 @@ import org.apache.hadoop.io.Text;
FilterLongScalarLessLongColumn.class, FilterLongScalarLessDoubleColumn.class,
FilterDoubleScalarLessLongColumn.class, FilterDoubleScalarLessDoubleColumn.class,
FilterDecimalColLessDecimalColumn.class, FilterDecimalColLessDecimalScalar.class,
- FilterDecimalScalarLessDecimalColumn.class})
+ FilterDecimalScalarLessDecimalColumn.class,
+ TimestampColLessTimestampScalar.class, TimestampScalarLessTimestampColumn.class,
+ FilterTimestampColLessTimestampScalar.class, FilterTimestampScalarLessTimestampColumn.class,
+ TimestampColLessLongScalar.class, LongScalarLessTimestampColumn.class,
+ FilterTimestampColLessLongScalar.class, FilterLongScalarLessTimestampColumn.class,
+ TimestampColLessDoubleScalar.class, DoubleScalarLessTimestampColumn.class,
+ FilterTimestampColLessDoubleScalar.class, FilterDoubleScalarLessTimestampColumn.class
+ })
public class GenericUDFOPLessThan extends GenericUDFBaseCompare {
public GenericUDFOPLessThan(){
this.opName = "LESS THAN";
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java Mon Oct 27 22:42:21 2014
@@ -49,7 +49,14 @@ import org.apache.hadoop.hive.serde2.obj
FilterLongScalarNotEqualLongColumn.class, FilterLongScalarNotEqualDoubleColumn.class,
FilterDoubleScalarNotEqualLongColumn.class, FilterDoubleScalarNotEqualDoubleColumn.class,
FilterDecimalColNotEqualDecimalColumn.class, FilterDecimalColNotEqualDecimalScalar.class,
- FilterDecimalScalarNotEqualDecimalColumn.class})
+ FilterDecimalScalarNotEqualDecimalColumn.class,
+ TimestampColNotEqualTimestampScalar.class, TimestampScalarNotEqualTimestampColumn.class,
+ FilterTimestampColNotEqualTimestampScalar.class, FilterTimestampScalarNotEqualTimestampColumn.class,
+ TimestampColNotEqualLongScalar.class, LongScalarNotEqualTimestampColumn.class,
+ FilterTimestampColNotEqualLongScalar.class, FilterLongScalarNotEqualTimestampColumn.class,
+ TimestampColNotEqualDoubleScalar.class, DoubleScalarNotEqualTimestampColumn.class,
+ FilterTimestampColNotEqualDoubleScalar.class, FilterDoubleScalarNotEqualTimestampColumn.class
+ })
public class GenericUDFOPNotEqual extends GenericUDFBaseCompare {
public GenericUDFOPNotEqual(){
this.opName = "NOT EQUAL";
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_13.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_13.q?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_13.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_13.q Mon Oct 27 22:42:21 2014
@@ -1,4 +1,102 @@
SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=minimal;
+
+EXPLAIN
+SELECT cboolean1,
+ ctinyint,
+ ctimestamp1,
+ cfloat,
+ cstring1,
+ (-(ctinyint)),
+ MAX(ctinyint),
+ ((-(ctinyint)) + MAX(ctinyint)),
+ SUM(cfloat),
+ (SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))),
+ (-(SUM(cfloat))),
+ (79.553 * cfloat),
+ STDDEV_POP(cfloat),
+ (-(SUM(cfloat))),
+ STDDEV_POP(ctinyint),
+ (((-(ctinyint)) + MAX(ctinyint)) - 10.175),
+ (-((-(SUM(cfloat))))),
+ (-26.28 / (-((-(SUM(cfloat)))))),
+ MAX(cfloat),
+ ((SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))) / ctinyint),
+ MIN(ctinyint)
+FROM alltypesorc
+WHERE (((cfloat < 3569)
+ AND ((10.175 >= cdouble)
+ AND (cboolean1 != 1)))
+ OR ((ctimestamp1 > 11)
+ AND ((ctimestamp2 != 12)
+ AND (ctinyint < 9763215.5639))))
+GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+LIMIT 40;
+
+SELECT cboolean1,
+ ctinyint,
+ ctimestamp1,
+ cfloat,
+ cstring1,
+ (-(ctinyint)),
+ MAX(ctinyint),
+ ((-(ctinyint)) + MAX(ctinyint)),
+ SUM(cfloat),
+ (SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))),
+ (-(SUM(cfloat))),
+ (79.553 * cfloat),
+ STDDEV_POP(cfloat),
+ (-(SUM(cfloat))),
+ STDDEV_POP(ctinyint),
+ (((-(ctinyint)) + MAX(ctinyint)) - 10.175),
+ (-((-(SUM(cfloat))))),
+ (-26.28 / (-((-(SUM(cfloat)))))),
+ MAX(cfloat),
+ ((SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))) / ctinyint),
+ MIN(ctinyint)
+FROM alltypesorc
+WHERE (((cfloat < 3569)
+ AND ((10.175 >= cdouble)
+ AND (cboolean1 != 1)))
+ OR ((ctimestamp1 > 11)
+ AND ((ctimestamp2 != 12)
+ AND (ctinyint < 9763215.5639))))
+GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+LIMIT 40;
+
+-- double compare timestamp
+EXPLAIN
+SELECT cboolean1,
+ ctinyint,
+ ctimestamp1,
+ cfloat,
+ cstring1,
+ (-(ctinyint)),
+ MAX(ctinyint),
+ ((-(ctinyint)) + MAX(ctinyint)),
+ SUM(cfloat),
+ (SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))),
+ (-(SUM(cfloat))),
+ (79.553 * cfloat),
+ STDDEV_POP(cfloat),
+ (-(SUM(cfloat))),
+ STDDEV_POP(ctinyint),
+ (((-(ctinyint)) + MAX(ctinyint)) - 10.175),
+ (-((-(SUM(cfloat))))),
+ (-26.28 / (-((-(SUM(cfloat)))))),
+ MAX(cfloat),
+ ((SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))) / ctinyint),
+ MIN(ctinyint)
+FROM alltypesorc
+WHERE (((cfloat < 3569)
+ AND ((10.175 >= cdouble)
+ AND (cboolean1 != 1)))
+ OR ((ctimestamp1 > -1.388)
+ AND ((ctimestamp2 != -1.3359999999999999)
+ AND (ctinyint < 9763215.5639))))
+GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+LIMIT 40;
+
SELECT cboolean1,
ctinyint,
ctimestamp1,
@@ -24,8 +122,9 @@ FROM alltypesorc
WHERE (((cfloat < 3569)
AND ((10.175 >= cdouble)
AND (cboolean1 != 1)))
- OR ((ctimestamp1 > -29071)
- AND ((ctimestamp2 != -29071)
+ OR ((ctimestamp1 > -1.388)
+ AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
-GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1;
+GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+LIMIT 40;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_7.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_7.q?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_7.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_7.q Mon Oct 27 22:42:21 2014
@@ -1,4 +1,7 @@
SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=minimal;
+
+EXPLAIN
SELECT cboolean1,
cbigint,
csmallint,
@@ -20,6 +23,84 @@ WHERE ((ctinyint != 0)
OR ((ctinyint = cint)
OR (cstring2 LIKE 'ss')))
AND ((988888 < cdouble)
- OR ((ctimestamp2 > -29071)
- AND (3569 >= cdouble)))));
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+LIMIT 25;
+
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint),
+ (csmallint % -257),
+ (-(csmallint)),
+ (-(ctinyint)),
+ ((-(ctinyint)) + 17),
+ (cbigint * (-(csmallint))),
+ (cint % csmallint),
+ (-(ctinyint)),
+ ((-(ctinyint)) % ctinyint)
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+LIMIT 25;
+
+-- double compare timestamp
+EXPLAIN
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint),
+ (csmallint % -257),
+ (-(csmallint)),
+ (-(ctinyint)),
+ ((-(ctinyint)) + 17),
+ (cbigint * (-(csmallint))),
+ (cint % csmallint),
+ (-(ctinyint)),
+ ((-(ctinyint)) % ctinyint)
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+LIMIT 25;
+
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint),
+ (csmallint % -257),
+ (-(csmallint)),
+ (-(ctinyint)),
+ ((-(ctinyint)) + 17),
+ (cbigint * (-(csmallint))),
+ (cint % csmallint),
+ (-(ctinyint)),
+ ((-(ctinyint)) % ctinyint)
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+LIMIT 25;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_8.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_8.q?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_8.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_8.q Mon Oct 27 22:42:21 2014
@@ -1,4 +1,7 @@
SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=minimal;
+
+EXPLAIN
SELECT ctimestamp1,
cdouble,
cboolean1,
@@ -15,9 +18,80 @@ SELECT ctimestamp1,
((-5638.15 - cdouble) + (cint + cfloat))
FROM alltypesorc
WHERE (((cstring2 IS NOT NULL)
- AND ((ctimestamp1 <= -29071)
- AND (ctimestamp2 != 16558)))
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
OR ((cfloat < -6432)
OR ((cboolean1 IS NOT NULL)
- AND (cdouble = 988888))));
+ AND (cdouble = 988888))))
+LIMIT 20;
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)),
+ (-5638.15 - cdouble),
+ (cdouble * -257),
+ (cint + cfloat),
+ ((-(cdouble)) + cbigint),
+ (-(cdouble)),
+ (-1.389 - cfloat),
+ (-(cfloat)),
+ ((-5638.15 - cdouble) + (cint + cfloat))
+FROM alltypesorc
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+LIMIT 20;
+
+-- double compare timestamp
+EXPLAIN
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)),
+ (-5638.15 - cdouble),
+ (cdouble * -257),
+ (cint + cfloat),
+ ((-(cdouble)) + cbigint),
+ (-(cdouble)),
+ (-1.389 - cfloat),
+ (-(cfloat)),
+ ((-5638.15 - cdouble) + (cint + cfloat))
+FROM alltypesorc
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+LIMIT 20;
+
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)),
+ (-5638.15 - cdouble),
+ (cdouble * -257),
+ (cint + cfloat),
+ ((-(cdouble)) + cbigint),
+ (-(cdouble)),
+ (-1.389 - cfloat),
+ (-(cfloat)),
+ ((-5638.15 - cdouble) + (cint + cfloat))
+FROM alltypesorc
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+LIMIT 20;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_short_regress.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_short_regress.q?rev=1634715&r1=1634714&r2=1634715&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_short_regress.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/vectorization_short_regress.q Mon Oct 27 22:42:21 2014
@@ -1,4 +1,5 @@
SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=minimal;
-- If you look at ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
-- which is the data generation class you'll see that those values are specified in the
@@ -58,7 +59,7 @@ EXPLAIN SELECT AVG(cint),
FROM alltypesorc
WHERE ((762 = cbigint)
OR ((csmallint < cfloat)
- AND ((ctimestamp2 > -10669)
+ AND ((ctimestamp2 > -5)
AND (cdouble != cint)))
OR (cstring1 = 'a')
OR ((cbigint <= -1.389)
@@ -93,7 +94,7 @@ SELECT AVG(cint),
FROM alltypesorc
WHERE ((762 = cbigint)
OR ((csmallint < cfloat)
- AND ((ctimestamp2 > -10669)
+ AND ((ctimestamp2 > -5)
AND (cdouble != cint)))
OR (cstring1 = 'a')
OR ((cbigint <= -1.389)
@@ -324,7 +325,9 @@ WHERE (((cstring1 RLIKE 'a.*')
OR ((cdouble > ctinyint)
AND (cfloat >= cint))
OR ((cint < cbigint)
- AND (ctinyint > cbigint)));
+ AND (ctinyint > cbigint)))
+LIMIT 50;
+
SELECT cint,
cdouble,
ctimestamp2,
@@ -357,7 +360,8 @@ WHERE (((cstring1 RLIKE 'a.*')
OR ((cdouble > ctinyint)
AND (cfloat >= cint))
OR ((cint < cbigint)
- AND (ctinyint > cbigint)));
+ AND (ctinyint > cbigint)))
+LIMIT 50;
-- TargetTypeClasses: Long, String, Double, Bool, Timestamp
-- Functions: VarP, Var, StDev, StDevP, Max, Sum
@@ -395,7 +399,8 @@ WHERE (((197 > ctinyint)
OR (cbigint = 359)
OR (cboolean1 < 0)
OR ((cstring1 LIKE '%ss')
- AND (cfloat <= ctinyint)));
+ AND (cfloat <= ctinyint)))
+LIMIT 25;
SELECT cint,
cbigint,
@@ -428,7 +433,8 @@ WHERE (((197 > ctinyint)
OR (cbigint = 359)
OR (cboolean1 < 0)
OR ((cstring1 LIKE '%ss')
- AND (cfloat <= ctinyint)));
+ AND (cfloat <= ctinyint)))
+LIMIT 25;
-- TargetTypeClasses: String, Bool, Double, Long, Timestamp
-- Functions: Sum, Max, Avg, Var, StDevP, VarP
@@ -466,7 +472,9 @@ WHERE (((csmallint > -26.28)
OR (ctinyint = -89010)
OR ((cbigint <= cfloat)
AND (-26.28 <= csmallint)))
-ORDER BY cboolean1, cstring1, ctimestamp2, cfloat, cbigint, cstring1, cdouble, cint, csmallint, cdouble;
+ORDER BY cboolean1, cstring1, ctimestamp2, cfloat, cbigint, cstring1, cdouble, cint, csmallint, cdouble
+LIMIT 75;
+
SELECT cint,
cstring1,
cboolean2,
@@ -498,7 +506,8 @@ WHERE (((csmallint > -26.28)
OR (ctinyint = -89010)
OR ((cbigint <= cfloat)
AND (-26.28 <= csmallint)))
-ORDER BY cboolean1, cstring1, ctimestamp2, cfloat, cbigint, cstring1, cdouble, cint, csmallint, cdouble;
+ORDER BY cboolean1, cstring1, ctimestamp2, cfloat, cbigint, cstring1, cdouble, cint, csmallint, cdouble
+LIMIT 75;
-- TargetTypeClasses: Long, String, Double, Timestamp
-- Functions: Avg, Min, StDevP, Sum, Var
@@ -529,7 +538,9 @@ WHERE (((-1.389 >= cint)
AND (cstring2 <= 'a'))
OR ((cstring1 LIKE 'ss%')
AND (10.175 > cbigint)))
-ORDER BY csmallint, cstring2, cdouble;
+ORDER BY csmallint, cstring2, cdouble
+LIMIT 45;
+
SELECT ctimestamp1,
cstring2,
cdouble,
@@ -554,7 +565,8 @@ WHERE (((-1.389 >= cint)
AND (cstring2 <= 'a'))
OR ((cstring1 LIKE 'ss%')
AND (10.175 > cbigint)))
-ORDER BY csmallint, cstring2, cdouble;
+ORDER BY csmallint, cstring2, cdouble
+LIMIT 45;
-- TargetTypeClasses: Double, String, Long
-- Functions: StDev, Sum, VarP, Count
@@ -578,7 +590,9 @@ WHERE (((csmallint >= -257))
OR ((cint >= cdouble)
AND (ctinyint <= cint))))
GROUP BY csmallint
-ORDER BY csmallint;
+ORDER BY csmallint
+LIMIT 20;
+
SELECT csmallint,
(csmallint % -75),
STDDEV_SAMP(csmallint),
@@ -596,7 +610,8 @@ WHERE (((csmallint >= -257))
OR ((cint >= cdouble)
AND (ctinyint <= cint))))
GROUP BY csmallint
-ORDER BY csmallint;
+ORDER BY csmallint
+LIMIT 20;
-- TargetTypeClasses: Long, Double, Timestamp
-- Functions: Var, Count, Sum, VarP, StDevP
@@ -705,14 +720,16 @@ WHERE (((ctimestamp1 != 0))
AND ((((-257 != ctinyint)
AND (cboolean2 IS NOT NULL))
AND ((cstring1 RLIKE '.*ss')
- AND (-10669 < ctimestamp1)))
- OR (ctimestamp2 = -10669)
+ AND (-3 < ctimestamp1)))
+ OR (ctimestamp2 = -5)
OR ((ctimestamp1 < 0)
AND (cstring2 LIKE '%b%'))
OR (cdouble = cint)
OR ((cboolean1 IS NULL)
AND (cfloat < cint))))
-GROUP BY ctimestamp1, cstring1;
+GROUP BY ctimestamp1, cstring1
+LIMIT 50;
+
SELECT ctimestamp1,
cstring1,
STDDEV_POP(cint),
@@ -757,14 +774,15 @@ WHERE (((ctimestamp1 != 0))
AND ((((-257 != ctinyint)
AND (cboolean2 IS NOT NULL))
AND ((cstring1 RLIKE '.*ss')
- AND (-10669 < ctimestamp1)))
- OR (ctimestamp2 = -10669)
+ AND (-3 < ctimestamp1)))
+ OR (ctimestamp2 = -5)
OR ((ctimestamp1 < 0)
AND (cstring2 LIKE '%b%'))
OR (cdouble = cint)
OR ((cboolean1 IS NULL)
AND (cfloat < cint))))
-GROUP BY ctimestamp1, cstring1;
+GROUP BY ctimestamp1, cstring1
+LIMIT 50;
-- TargetTypeClasses: Double, Long, String, Timestamp, Bool
-- Functions: Max, Sum, Var, Avg, Min, VarP, StDev, StDevP