You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2013/05/22 23:04:35 UTC
svn commit: r1485421 [1/6] - in /hive/branches/vectorization/ql/src:
java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/
test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/
test/org/apache/hadoop/hive/ql/exec/vector/util/
Author: omalley
Date: Wed May 22 21:04:35 2013
New Revision: 1485421
URL: http://svn.apache.org/r1485421
Log:
HIVE-4553 Column Column, and Column Scalar vectorized execution tests
(Tony Murphy via omalley)
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnFilterVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnColumnOperationVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarFilterVectorExpressionEvaluation.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/TestColumnScalarOperationVectorExpressionEvaluation.java
Modified:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java?rev=1485421&r1=1485420&r2=1485421&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java Wed May 22 21:04:35 2013
@@ -216,10 +216,12 @@ public class CodeGen {
};
+
private final String templateDirectory;
private final String outputDirectory;
+ private final TestCodeGen testCodeGen;
- private static String joinPath(String...parts) {
+ static String joinPath(String...parts) {
String path = parts[0];
for (int i=1; i < parts.length; ++i) {
path += File.separatorChar + parts[i];
@@ -231,11 +233,13 @@ public class CodeGen {
templateDirectory = System.getProperty("user.dir");
File f = new File(templateDirectory);
outputDirectory = joinPath(f.getParent(), "gen");
+ testCodeGen = new TestCodeGen(joinPath(f.getParent(), "test"),templateDirectory);
}
- public CodeGen(String templateDirectory, String outputDirectory) {
+ public CodeGen(String templateDirectory, String outputDirectory, String testOutputDirectory) {
this.templateDirectory = templateDirectory;
this.outputDirectory = outputDirectory;
+ testCodeGen = new TestCodeGen(testOutputDirectory,templateDirectory);
}
/**
@@ -244,10 +248,13 @@ public class CodeGen {
*/
public static void main(String[] args) throws Exception {
CodeGen gen;
- if (args != null && args[0] != null) {
- gen = new CodeGen(args[0], args[1]);
- } else {
+ if (args == null || args.length==0) {
gen = new CodeGen();
+ } else if (args.length==3) {
+ gen = new CodeGen(args[0], args[1], args[2]);
+ }else{
+ System.out.println("args: <templateDir> <outputDir> <testOutputDir>");
+ return;
}
gen.generate();
}
@@ -287,6 +294,7 @@ public class CodeGen {
continue;
}
}
+ testCodeGen.generateTestSuites();
}
private void generateVectorUDAFMinMax(String[] tdesc) throws Exception {
@@ -417,9 +425,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Column";
- generateColumnBinaryOperatorColumn(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
+ generateColumnBinaryOperatorColumn(tdesc, null, className);
}
private void generateColumnUnaryMinus(String[] tdesc) throws IOException {
@@ -427,7 +435,7 @@ public class CodeGen {
String inputColumnVectorType = this.getColumnVectorType(operandType);
String outputColumnVectorType = inputColumnVectorType;
String returnType = operandType;
- String className = this.getCamelCaseType(operandType) + "ColUnaryMinus";
+ String className = getCamelCaseType(operandType) + "ColUnaryMinus";
String outputFile = joinPath(this.outputDirectory, className + ".java");
String templateFile = joinPath(this.templateDirectory, tdesc[0] + ".txt");
String templateString = readFile(templateFile);
@@ -444,8 +452,8 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Column";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateColumnBinaryOperatorColumn(tdesc, returnType, className);
}
@@ -456,9 +464,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar";
- generateColumnBinaryOperatorScalar(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
+ generateColumnBinaryOperatorScalar(tdesc, null, className);
}
private void generateFilterScalarCompareColumn(String[] tdesc) throws IOException {
@@ -466,9 +474,9 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = "Filter" + this.getCamelCaseType(operandType1)
- + "Scalar" + operatorName + this.getCamelCaseType(operandType2) + "Column";
- generateScalarBinaryOperatorColumn(tdesc, "doesn't matter", className);
+ String className = "Filter" + getCamelCaseType(operandType1)
+ + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
+ generateScalarBinaryOperatorColumn(tdesc, null, className);
}
private void generateColumnCompareScalar(String[] tdesc) throws IOException {
@@ -476,8 +484,8 @@ public class CodeGen {
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String returnType = "long";
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
generateColumnBinaryOperatorScalar(tdesc, returnType, className);
}
@@ -503,6 +511,20 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null){
+ testCodeGen.addColumnColumnFilterTestCases(
+ className,
+ inputColumnVectorType1,
+ inputColumnVectorType2,
+ operatorSymbol);
+ }else{
+ testCodeGen.addColumnColumnOperationTestCases(
+ className,
+ inputColumnVectorType1,
+ inputColumnVectorType2,
+ outputColumnVectorType);
+ }
}
private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType,
@@ -525,6 +547,25 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null)
+ {
+ testCodeGen.addColumnScalarFilterTestCases(
+ true,
+ className,
+ inputColumnVectorType,
+ operandType2,
+ operatorSymbol);
+ }else
+ {
+ testCodeGen.addColumnScalarOperationTestCases(
+ true,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType2);
+ }
+
}
private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType,
@@ -547,6 +588,24 @@ public class CodeGen {
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
writeFile(outputFile, templateString);
+
+ if(returnType==null)
+ {
+ testCodeGen.addColumnScalarFilterTestCases(
+ false,
+ className,
+ inputColumnVectorType,
+ operandType1,
+ operatorSymbol);
+ }else
+ {
+ testCodeGen.addColumnScalarOperationTestCases(
+ false,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType1);
+ }
}
//Binary arithmetic operator
@@ -554,8 +613,8 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Col" + operatorName + this.getCamelCaseType(operandType2) + "Scalar";
+ String className = getCamelCaseType(operandType1)
+ + "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateColumnBinaryOperatorScalar(tdesc, returnType, className);
}
@@ -564,19 +623,20 @@ public class CodeGen {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String className = this.getCamelCaseType(operandType1)
- + "Scalar" + operatorName + this.getCamelCaseType(operandType2) + "Column";
+ String className = getCamelCaseType(operandType1)
+ + "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
generateScalarBinaryOperatorColumn(tdesc, returnType, className);
}
- private void writeFile(String outputFile, String str) throws IOException {
+
+ static void writeFile(String outputFile, String str) throws IOException {
BufferedWriter w = new BufferedWriter(new FileWriter(outputFile));
w.write(str);
w.close();
}
- private String readFile(String templateFile) throws IOException {
+ static String readFile(String templateFile) throws IOException {
BufferedReader r = new BufferedReader(new FileReader(templateFile));
String line = r.readLine();
StringBuilder b = new StringBuilder();
@@ -589,7 +649,7 @@ public class CodeGen {
return b.toString();
}
- private String getCamelCaseType(String type) {
+ static String getCamelCaseType(String type) {
if (type.equals("long")) {
return "Long";
} else if (type.equals("double")) {
@@ -610,9 +670,7 @@ public class CodeGen {
}
private String getColumnVectorType(String primitiveType) {
- if (primitiveType.equals("long")) {
- return "LongColumnVector";
- } else if (primitiveType.equals("double")) {
+ if(primitiveType!=null && primitiveType.equals("double")) {
return "DoubleColumnVector";
}
return "LongColumnVector";
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestClass.txt Wed May 22 21:04:35 2013
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import static org.junit.Assert.assertEquals;
+import java.util.Random;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil;
+import org.junit.Test;
+
+
+/**
+ *
+ * <ClassName>.
+ *
+ */
+public class <ClassName>{
+
+ private static final int BATCH_SIZE = 100;
+ private static final long SEED = 0xfa57;
+
+ <TestCases>
+
+}
+
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestCodeGen.java Wed May 22 21:04:35 2013
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.templates;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ *
+ * TestCodeGen.
+ * This class is mutable and maintains a hashmap of TestSuiteClassName to test cases.
+ * The tests cases are added over the course of vectorized expressions class generation,
+ * with test classes being outputted at the end. For each column vector (inputs and/or outputs)
+ * a matrix of pairwise covering Booleans is used to generate test cases across nulls and
+ * repeating dimensions. Based on the input column vector(s) nulls and repeating states
+ * the states of the output column vector (if there is one) is validated, along with the null
+ * vector. For filter operations the selection vector is validated against the generated
+ * data. Each template corresponds to a class representing a test suite.
+ */
+public class TestCodeGen {
+
+ public enum TestSuiteClassName{
+ TestColumnScalarOperationVectorExpressionEvaluation,
+ TestColumnScalarFilterVectorExpressionEvaluation,
+ TestColumnColumnOperationVectorExpressionEvaluation,
+ TestColumnColumnFilterVectorExpressionEvaluation,
+ }
+
+ private final String testOutputDir;
+ private final String templateDirectory;
+ private final HashMap<TestSuiteClassName,StringBuilder> testsuites;
+
+ public TestCodeGen(String testOutputDir, String templateDirectory) {
+ this.testOutputDir = testOutputDir;
+ this.templateDirectory = templateDirectory;
+ testsuites = new HashMap<TestSuiteClassName, StringBuilder>();
+
+ for(TestSuiteClassName className : TestSuiteClassName.values()) {
+ testsuites.put(className,new StringBuilder());
+ }
+
+ }
+
+ public void addColumnScalarOperationTestCases(boolean op1IsCol, String vectorExpClassName,
+ String inputColumnVectorType, String outputColumnVectorType, String scalarType)
+ throws IOException {
+
+ TestSuiteClassName template =
+ TestSuiteClassName.TestColumnScalarOperationVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString = removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix :new Boolean[][]{
+ // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, ColumnHasNulls, ColumnIsRepeating
+ {false, true, true, true},
+ {false, false, false, false},
+ {true, false, true, false},
+ {true, true, false, false},
+ {true, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1])
+ + createNullRepeatingNameFragment("Col", testMatrix[2], testMatrix[3]));
+ testCase = testCase.replaceAll("<VectorExpClassName>", vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
+ testCase = testCase.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ testCase = testCase.replaceAll("<ScalarType>", scalarType);
+ testCase = testCase.replaceAll("<CamelCaseScalarType>", CodeGen.getCamelCaseType(scalarType));
+ testCase = testCase.replaceAll("<InitOuputColHasNulls>", testMatrix[0].toString());
+ testCase = testCase.replaceAll("<InitOuputColIsRepeating>", testMatrix[1].toString());
+ testCase = testCase.replaceAll("<ColumnHasNulls>", testMatrix[2].toString());
+ testCase = testCase.replaceAll("<ColumnIsRepeating>", testMatrix[3].toString());
+
+ if(op1IsCol){
+ testCase = testCase.replaceAll("<ConstructorParams>","0, scalarValue");
+ }else{
+ testCase = testCase.replaceAll("<ConstructorParams>","scalarValue, 0");
+ }
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnScalarFilterTestCases(boolean op1IsCol, String vectorExpClassName,
+ String inputColumnVectorType, String scalarType, String operatorSymbol)
+ throws IOException {
+
+ TestSuiteClassName template =
+ TestSuiteClassName.TestColumnScalarFilterVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString = removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: ColumnHasNulls, ColumnIsRepeating
+ {true, true},
+ {true, false},
+ {false, false},
+ {false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Col", testMatrix[0], testMatrix[1]));
+ testCase = testCase.replaceAll("<VectorExpClassName>", vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
+ testCase = testCase.replaceAll("<ScalarType>", scalarType);
+ testCase = testCase.replaceAll("<CamelCaseScalarType>", CodeGen.getCamelCaseType(scalarType));
+ testCase = testCase.replaceAll("<ColumnHasNulls>", testMatrix[0].toString());
+ testCase = testCase.replaceAll("<ColumnIsRepeating>", testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Operator>", operatorSymbol);
+
+ if(op1IsCol){
+ testCase = testCase.replaceAll("<ConstructorParams>","0, scalarValue");
+ }else{
+ testCase = testCase.replaceAll("<ConstructorParams>","scalarValue, 0");
+ }
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnColumnOperationTestCases(String vectorExpClassName,
+ String inputColumnVectorType1, String inputColumnVectorType2, String outputColumnVectorType)
+ throws IOException {
+
+ TestSuiteClassName template=
+ TestSuiteClassName.TestColumnColumnOperationVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString = removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: InitOuputColHasNulls, InitOuputColIsRepeating, Column1HasNulls,
+ // Column1IsRepeating, Column2HasNulls, Column2IsRepeating
+ {true, true, false, true, true, true},
+ {false, false, true, false, false, false},
+ {true, false, true, false, true, true},
+ {true, true, true, true, false, false},
+ {false, false, false, true, true, false},
+ {false, true, false, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("Out", testMatrix[0], testMatrix[1])
+ + createNullRepeatingNameFragment("C1", testMatrix[2], testMatrix[3])
+ + createNullRepeatingNameFragment("C2", testMatrix[4], testMatrix[5]));
+ testCase = testCase.replaceAll("<VectorExpClassName>", vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
+ testCase = testCase.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+ testCase = testCase.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ testCase = testCase.replaceAll("<InitOuputColHasNulls>", testMatrix[0].toString());
+ testCase = testCase.replaceAll("<InitOuputColIsRepeating>", testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Column1HasNulls>", testMatrix[2].toString());
+ testCase = testCase.replaceAll("<Column1IsRepeating>", testMatrix[3].toString());
+ testCase = testCase.replaceAll("<Column2HasNulls>", testMatrix[4].toString());
+ testCase = testCase.replaceAll("<Column2IsRepeating>", testMatrix[5].toString());
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void addColumnColumnFilterTestCases(String vectorExpClassName,
+ String inputColumnVectorType1, String inputColumnVectorType2, String operatorSymbol)
+ throws IOException {
+
+ TestSuiteClassName template=
+ TestSuiteClassName.TestColumnColumnFilterVectorExpressionEvaluation;
+
+ //Read the template into a string;
+ String templateFile = CodeGen.joinPath(this.templateDirectory,template.toString()+".txt");
+ String templateString = removeTemplateComments(CodeGen.readFile(templateFile));
+
+ for(Boolean[] testMatrix : new Boolean[][]{
+ // Pairwise: Column1HasNulls, Column1IsRepeating, Column2HasNulls, Column2IsRepeating
+ {false, true, true, true},
+ {false, false, false, false},
+ {true, false, true, false},
+ {true, true, false, false},
+ {true, false, false, true}}) {
+ String testCase = templateString;
+ testCase = testCase.replaceAll("<TestName>",
+ "test"
+ + vectorExpClassName
+ + createNullRepeatingNameFragment("C1", testMatrix[0], testMatrix[1])
+ + createNullRepeatingNameFragment("C2", testMatrix[2], testMatrix[3]));
+ testCase = testCase.replaceAll("<VectorExpClassName>", vectorExpClassName);
+ testCase = testCase.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
+ testCase = testCase.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+ testCase = testCase.replaceAll("<Column1HasNulls>", testMatrix[0].toString());
+ testCase = testCase.replaceAll("<Column1IsRepeating>", testMatrix[1].toString());
+ testCase = testCase.replaceAll("<Column2HasNulls>", testMatrix[2].toString());
+ testCase = testCase.replaceAll("<Column2IsRepeating>", testMatrix[3].toString());
+ testCase = testCase.replaceAll("<Operator>", operatorSymbol);
+
+ testsuites.get(template).append(testCase);
+ }
+ }
+
+ public void generateTestSuites() throws IOException {
+
+ String templateFile = CodeGen.joinPath(this.templateDirectory, "TestClass.txt");
+ for(TestSuiteClassName testClass : testsuites.keySet()) {
+
+ String templateString = CodeGen.readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", testClass.toString());
+ templateString = templateString.replaceAll("<TestCases>", testsuites.get(testClass).toString());
+
+ String outputFile = CodeGen.joinPath(this.testOutputDir, testClass + ".java");
+
+ CodeGen.writeFile(outputFile, templateString);
+ }
+ }
+
+ private static String createNullRepeatingNameFragment(String idenitfier, boolean nulls, boolean repeating)
+ {
+ if(nulls || repeating){
+ if(nulls){
+ idenitfier+="Nulls";
+ }
+ if(repeating){
+ idenitfier+="Repeats";
+ }
+ return idenitfier;
+ }
+
+ return "";
+ }
+
+ private static String removeTemplateComments(String templateString){
+ return templateString.replaceAll("(?s)<!--(.*)-->", "");
+ }
+}
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnFilterVectorExpressionEvaluation.txt Wed May 22 21:04:35 2013
@@ -0,0 +1,69 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <InputColumnVectorType1> inputColumnVector1 =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType1>(<Column1HasNulls>,
+ <Column1IsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType2> inputColumnVector2 =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType2>(<Column2HasNulls>,
+ <Column2IsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector1;
+ rowBatch.cols[1] = inputColumnVector2;
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(0, 1);
+
+ vectorExpression.evaluate(rowBatch);
+
+ int selectedIndex = 0;
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vector is safe to check, as it is always initialized to match the data vector
+ if(!inputColumnVector1.isNull[i] && !inputColumnVector2.isNull[i]) {
+ if(inputColumnVector1.vector[i] <Operator> inputColumnVector2.vector[i]) {
+ assertEquals(
+ "Vector index that passes filter "
+ + inputColumnVector1.vector[i] + "<Operator>"
+ + inputColumnVector2.vector[i] + " is not in rowBatch selected index",
+ i,
+ rowBatch.selected[selectedIndex]);
+ selectedIndex++;
+ }
+ }
+ }
+
+ assertEquals("Row batch size not set to number of selected rows: " + selectedIndex,
+ selectedIndex, rowBatch.size);
+
+ if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ true, rowBatch.selectedInUse);
+ } else if(selectedIndex == BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ false, rowBatch.selectedInUse);
+ }
+ }
\ No newline at end of file
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnColumnOperationVectorExpressionEvaluation.txt Wed May 22 21:04:35 2013
@@ -0,0 +1,64 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <OutputColumnVectorType> outputColumnVector =
+ VectorizedRowGroupGenUtil.generate<OutputColumnVectorType>(<InitOuputColHasNulls>,
+ <InitOuputColIsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType1> inputColumnVector1 =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType1>(<Column1HasNulls>,
+ <Column1IsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType2> inputColumnVector2 =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType2>(<Column2HasNulls>,
+ <Column2IsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(3, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector1;
+ rowBatch.cols[1] = inputColumnVector2;
+ rowBatch.cols[2] = outputColumnVector;
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(0, 1, 2);
+
+ vectorExpression.evaluate(rowBatch);
+
+ assertEquals(
+ "Output column vector repeating state does not match operand columns",
+ (!inputColumnVector1.noNulls && inputColumnVector1.isRepeating)
+ || (!inputColumnVector2.noNulls && inputColumnVector2.isRepeating)
+ || inputColumnVector1.isRepeating && inputColumnVector2.isRepeating,
+ outputColumnVector.isRepeating);
+
+ assertEquals(
+ "Output column vector no nulls state does not match operand columns",
+ inputColumnVector1.noNulls && inputColumnVector2.noNulls, outputColumnVector.noNulls);
+
+ //if repeating, only the first value matters
+ if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vectors are safe to check, as they are always initialized to match the data vector
+ assertEquals("Output vector doesn't match input vectors' is null state for index",
+ inputColumnVector1.isNull[i] || inputColumnVector2.isNull[i],
+ outputColumnVector.isNull[i]);
+ }
+ }
+ }
\ No newline at end of file
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarFilterVectorExpressionEvaluation.txt Wed May 22 21:04:35 2013
@@ -0,0 +1,76 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <InputColumnVectorType> inputColumnVector =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType>(<ColumnHasNulls>,
+ <ColumnIsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(1, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector;
+
+ <ScalarType> scalarValue = 0;
+ do {
+ scalarValue = rand.next<CamelCaseScalarType>();
+ } while(scalarValue == 0);
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(<ConstructorParams>);
+
+ vectorExpression.evaluate(rowBatch);
+
+ int selectedIndex = 0;
+ //check for isRepeating optimization
+ if(inputColumnVector.isRepeating) {
+ //null vector is safe to check, as it is always initialized to match the data vector
+ selectedIndex =
+ !inputColumnVector.isNull[0] && inputColumnVector.vector[0] <Operator> scalarValue
+ ? BATCH_SIZE : 0;
+ } else {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ if(!inputColumnVector.isNull[i]) {
+ if(inputColumnVector.vector[i] <Operator> scalarValue) {
+ assertEquals(
+ "Vector index that passes filter "
+ + inputColumnVector.vector[i] + "<Operator>"
+ + scalarValue + " is not in rowBatch selected index",
+ i,
+ rowBatch.selected[selectedIndex]);
+ selectedIndex++;
+ }
+ }
+ }
+ }
+
+ assertEquals("Row batch size not set to number of selected rows: " + selectedIndex,
+ selectedIndex, rowBatch.size);
+
+ if(selectedIndex > 0 && selectedIndex < BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should be set when > 0 and < entire batch(" + BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ true, rowBatch.selectedInUse);
+ } else if(selectedIndex == BATCH_SIZE) {
+ assertEquals(
+ "selectedInUse should not be set when entire batch(" + BATCH_SIZE + ") is selected: "
+ + selectedIndex,
+ false, rowBatch.selectedInUse);
+ }
+ }
\ No newline at end of file
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt?rev=1485421&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/TestColumnScalarOperationVectorExpressionEvaluation.txt Wed May 22 21:04:35 2013
@@ -0,0 +1,59 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ @Test
+ public void <TestName>() {
+
+ Random rand = new Random(SEED);
+
+ <OutputColumnVectorType> outputColumnVector =
+ VectorizedRowGroupGenUtil.generate<OutputColumnVectorType>(<InitOuputColHasNulls>,
+ <InitOuputColIsRepeating>, BATCH_SIZE, rand);
+
+ <InputColumnVectorType> inputColumnVector =
+ VectorizedRowGroupGenUtil.generate<InputColumnVectorType>(<ColumnHasNulls>,
+ <ColumnIsRepeating>, BATCH_SIZE, rand);
+
+ VectorizedRowBatch rowBatch = new VectorizedRowBatch(2, BATCH_SIZE);
+ rowBatch.cols[0] = inputColumnVector;
+ rowBatch.cols[1] = outputColumnVector;
+
+ <ScalarType> scalarValue = 0;
+ do {
+ scalarValue = rand.next<CamelCaseScalarType>();
+ } while(scalarValue == 0);
+
+ <VectorExpClassName> vectorExpression =
+ new <VectorExpClassName>(<ConstructorParams>, 1);
+
+ vectorExpression.evaluate(rowBatch);
+
+ assertEquals(
+ "Output column vector is repeating state does not match operand column",
+ inputColumnVector.isRepeating, outputColumnVector.isRepeating);
+
+ assertEquals(
+ "Output column vector no nulls state does not match operand column",
+ inputColumnVector.noNulls, outputColumnVector.noNulls);
+
+ if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
+ for(int i = 0; i < BATCH_SIZE; i++) {
+ //null vectors are safe to check, as they are always initialized to match the data vector
+ assertEquals("Output vector doesn't match input vector's is null state for index",
+ inputColumnVector.isNull[i], outputColumnVector.isNull[i]);
+ }
+ }
+ }
\ No newline at end of file