You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by eh...@apache.org on 2013/12/12 20:01:10 UTC
svn commit: r1550488 [1/2] - in /hive/trunk:
ant/src/org/apache/hadoop/hive/ant/
ql/src/gen/vectorization/ExpressionTemplates/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ ql/src/java...
Author: ehans
Date: Thu Dec 12 19:01:09 2013
New Revision: 1550488
URL: http://svn.apache.org/r1550488
Log:
HIVE-5756: Implement vectorized support for IF conditional expression (Eric Hanson)
Added:
hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt
hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt
hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt
hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringColumn.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringScalar.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringColumn.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorConditionalExpressions.java
Modified:
hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatch.java
Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Thu Dec 12 19:01:09 2013
@@ -386,9 +386,25 @@ public class GenVectorCode extends Task
// See org.apache.hadoop.hive.ql.exec.vector.expressions for remaining cast VectorExpression
// classes
- {"ColumnUnaryMinus", "long"},
- {"ColumnUnaryMinus", "double"},
+ {"ColumnUnaryMinus", "long"},
+ {"ColumnUnaryMinus", "double"},
+ // IF conditional expression
+ // fileHeader, resultType, arg2Type, arg3Type
+ {"IfExprColumnColumn", "long"},
+ {"IfExprColumnColumn", "double"},
+ {"IfExprColumnScalar", "long", "long"},
+ {"IfExprColumnScalar", "double", "long"},
+ {"IfExprColumnScalar", "long", "double"},
+ {"IfExprColumnScalar", "double", "double"},
+ {"IfExprScalarColumn", "long", "long"},
+ {"IfExprScalarColumn", "double", "long"},
+ {"IfExprScalarColumn", "long", "double"},
+ {"IfExprScalarColumn", "double", "double"},
+ {"IfExprScalarScalar", "long", "long"},
+ {"IfExprScalarScalar", "double", "long"},
+ {"IfExprScalarScalar", "long", "double"},
+ {"IfExprScalarScalar", "double", "double"},
// template, <ClassName>, <ValueType>, <OperatorSymbol>, <DescriptionName>, <DescriptionValue>
{"VectorUDAFMinMax", "VectorUDAFMinLong", "long", "<", "min",
@@ -567,6 +583,14 @@ public class GenVectorCode extends Task
generateFilterStringColumnCompareColumn(tdesc);
} else if (tdesc[0].equals("StringColumnCompareColumn")) {
generateStringColumnCompareColumn(tdesc);
+ } else if (tdesc[0].equals("IfExprColumnColumn")) {
+ generateIfExprColumnColumn(tdesc);
+ } else if (tdesc[0].equals("IfExprColumnScalar")) {
+ generateIfExprColumnScalar(tdesc);
+ } else if (tdesc[0].equals("IfExprScalarColumn")) {
+ generateIfExprScalarColumn(tdesc);
+ } else if (tdesc[0].equals("IfExprScalarScalar")) {
+ generateIfExprScalarScalar(tdesc);
} else {
continue;
}
@@ -800,6 +824,89 @@ public class GenVectorCode extends Task
className, templateString);
}
+ private void generateIfExprColumnColumn(String[] tdesc) throws IOException {
+ String operandType = tdesc[1];
+ String inputColumnVectorType = this.getColumnVectorType(operandType);
+ String outputColumnVectorType = inputColumnVectorType;
+ String returnType = operandType;
+ String className = "IfExpr" + getCamelCaseType(operandType) + "Column"
+ + getCamelCaseType(operandType) + "Column";
+ String outputFile = joinPath(this.expressionOutputDirectory, className + ".java");
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
+ templateString = templateString.replaceAll("<OperandType>", operandType);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateIfExprColumnScalar(String[] tdesc) throws IOException {
+ String operandType2 = tdesc[1];
+ String operandType3 = tdesc[2];
+ String arg2ColumnVectorType = this.getColumnVectorType(operandType2);
+ String returnType = getArithmeticReturnType(operandType2, operandType3);
+ String outputColumnVectorType = getColumnVectorType(returnType);
+ String className = "IfExpr" + getCamelCaseType(operandType2) + "Column"
+ + getCamelCaseType(operandType3) + "Scalar";
+ String outputFile = joinPath(this.expressionOutputDirectory, className + ".java");
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<Arg2ColumnVectorType>", arg2ColumnVectorType);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<OperandType3>", operandType3);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateIfExprScalarColumn(String[] tdesc) throws IOException {
+ String operandType2 = tdesc[1];
+ String operandType3 = tdesc[2];
+ String arg3ColumnVectorType = this.getColumnVectorType(operandType3);
+ String returnType = getArithmeticReturnType(operandType2, operandType3);
+ String outputColumnVectorType = getColumnVectorType(returnType);
+ String className = "IfExpr" + getCamelCaseType(operandType2) + "Scalar"
+ + getCamelCaseType(operandType3) + "Column";
+ String outputFile = joinPath(this.expressionOutputDirectory, className + ".java");
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<Arg3ColumnVectorType>", arg3ColumnVectorType);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<OperandType3>", operandType3);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateIfExprScalarScalar(String[] tdesc) throws IOException {
+ String operandType2 = tdesc[1];
+ String operandType3 = tdesc[2];
+ String arg3ColumnVectorType = this.getColumnVectorType(operandType3);
+ String returnType = getArithmeticReturnType(operandType2, operandType3);
+ String outputColumnVectorType = getColumnVectorType(returnType);
+ String className = "IfExpr" + getCamelCaseType(operandType2) + "Scalar"
+ + getCamelCaseType(operandType3) + "Scalar";
+ String outputFile = joinPath(this.expressionOutputDirectory, className + ".java");
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<OperandType3>", operandType3);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
// template, <ClassNamePrefix>, <ReturnType>, <OperandType>, <FuncName>, <OperandCast>, <ResultCast>
private void generateColumnUnaryFunc(String[] tdesc) throws IOException {
String classNamePrefix = tdesc[1];
Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnColumn.txt Thu Dec 12 19:01:09 2013
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second and third are long columns or long expression results.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg2Column, arg3Column;
+ private int outputColumn;
+
+ public <ClassName>(int arg1Column, int arg2Column, int arg3Column, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ <InputColumnVectorType> arg2ColVector = (<InputColumnVectorType>) batch.cols[arg2Column];
+ <InputColumnVectorType> arg3ColVector = (<InputColumnVectorType>) batch.cols[arg3Column];
+ <InputColumnVectorType> outputColVector = (<InputColumnVectorType>) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls;
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+ <OperandType>[] vector2 = arg2ColVector.vector;
+ <OperandType>[] vector3 = arg3ColVector.vector;
+ <OperandType>[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ /* All the code paths below propagate nulls even if neither arg2 nor arg3
+ * have nulls. This is to reduce the number of code paths and shorten the
+ * code, at the expense of maybe doing unnecessary work if neither input
+ * has nulls. This could be improved in the future by expanding the number
+ * of code paths.
+ */
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // extend any repeating values and noNulls indicator in the inputs
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]);
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (vector1[i] == 1 ? vector2[i] : vector3[i]);
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ vector2[i] : vector3[i]);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ vector2[i] : vector3[i]);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg2ColVector.unFlatten();
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<OperandType>";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public int getArg2Column() {
+ return arg2Column;
+ }
+
+ public void setArg2Column(int colNum) {
+ this.arg2Column = colNum;
+ }
+
+ public int getArg3Column() {
+ return arg3Column;
+ }
+
+ public void setArg3Column(int colNum) {
+ this.arg3Column = colNum;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType>"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}
Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprColumnScalar.txt Thu Dec 12 19:01:09 2013
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a column or non-constant expression result.
+ * The third is a constant value.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg2Column;
+ private <OperandType3> arg3Scalar;
+ private int outputColumn;
+
+ public <ClassName>(int arg1Column, int arg2Column, <OperandType3> arg3Scalar,
+ int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ <Arg2ColumnVectorType> arg2ColVector = (<Arg2ColumnVectorType>) batch.cols[arg2Column];
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls; // nulls can only come from arg2
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+ <OperandType2>[] vector2 = arg2ColVector.vector;
+ <ReturnType>[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ return;
+ }
+
+ // Extend any repeating values and noNulls indicator in the inputs to
+ // reduce the number of code paths needed below.
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (vector1[i] == 1 ? vector2[i] : arg3Scalar);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (vector1[i] == 1 ? vector2[i] : arg3Scalar);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ vector2[i] : arg3Scalar);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ vector2[i] : arg3Scalar);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg2ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public int getArg2Column() {
+ return arg2Column;
+ }
+
+ public void setArg2Column(int colNum) {
+ this.arg2Column = colNum;
+ }
+
+ public <OperandType3> getArg3Scalar() {
+ return arg3Scalar;
+ }
+
+ public void setArg3Scalar(<OperandType3> value) {
+ this.arg3Scalar = value;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType3>"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}
Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarColumn.txt Thu Dec 12 19:01:09 2013
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a column or non-constant expression result.
+ * The third is a constant value.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg3Column;
+ private <OperandType2> arg2Scalar;
+ private int outputColumn;
+
+ public <ClassName>(int arg1Column, <OperandType2> arg2Scalar, int arg3Column,
+ int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ <Arg3ColumnVectorType> arg3ColVector = (<Arg3ColumnVectorType>) batch.cols[arg3Column];
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg3ColVector.noNulls; // nulls can only come from arg3 column vector
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+ <OperandType3>[] vector3 = arg3ColVector.vector;
+ <ReturnType>[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // Extend any repeating values and noNulls indicator in the inputs to
+ // reduce the number of code paths needed below.
+ // This could be optimized in the future by having separate paths
+ // for when arg3ColVector is repeating or has no nulls.
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (vector1[i] == 1 ? arg2Scalar : vector3[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (vector1[i] == 1 ? arg2Scalar : vector3[i]);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : vector3[i]);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : vector3[i]);
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+
+ // restore repeating and no nulls indicators
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public int getArg3Column() {
+ return arg3Column;
+ }
+
+ public void setArg3Column(int colNum) {
+ this.arg3Column = colNum;
+ }
+
+ public <OperandType2> getArg2Scalar() {
+ return arg2Scalar;
+ }
+
+ public void setArg2Scalar(<OperandType2> value) {
+ this.arg2Scalar = value;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType3>"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}
Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt (added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/IfExprScalarScalar.txt Thu Dec 12 19:01:09 2013
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import java.util.Arrays;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a constant value.
+ * The third is a constant value.
+ */
+public class <ClassName> extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column;
+ private <OperandType2> arg2Scalar;
+ private <OperandType3> arg3Scalar;
+ private int outputColumn;
+
+ public <ClassName>(int arg1Column, <OperandType2> arg2Scalar, <OperandType3> arg3Scalar,
+ int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
+ }
+
+ public <ClassName>() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = false; // output is a scalar which we know is non null
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+ <ReturnType>[] outputVector = outputColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ } else if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (vector1[i] == 1 ? arg2Scalar : arg3Scalar);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (vector1[i] == 1 ? arg2Scalar : arg3Scalar);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3Scalar);
+ outputIsNull[i] = false;
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ outputVector[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2Scalar : arg3Scalar);
+ }
+ Arrays.fill(outputIsNull, 0, n, false);
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "<ReturnType>";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public <OperandType2> getArg2Scalar() {
+ return arg2Scalar;
+ }
+
+ public void setArg2Scalar(<OperandType2> value) {
+ this.arg2Scalar = value;
+ }
+
+ public <OperandType3> getArg3Scalar() {
+ return arg3Scalar;
+ }
+
+ public void setArg3Scalar(<OperandType3> value) {
+ this.arg3Scalar = value;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType2>"),
+ VectorExpressionDescriptor.ArgumentType.getType("<OperandType3>"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java Thu Dec 12 19:01:09 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
+
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -219,4 +221,93 @@ public class BytesColumnVector extends C
}
return result;
}
+
+ /** Copy the current object contents into the output. Only copy selected entries,
+ * as indicated by selectedInUse and the sel array.
+ */
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, BytesColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.setVal(0, vector[0], start[0], length[0]);
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.setVal(i, vector[i], start[i], length[i]);
+ }
+ }
+ else {
+ for (int i = 0; i < size; i++) {
+ output.setVal(i, vector[i], start[i], length[i]);
+ }
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /** Simplify vector by brute-force flattening noNulls and isRepeating
+ * This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ * with many arguments, at the expense of loss of some performance.
+ */
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+
+ // setRef is used below and this is safe, because the reference
+ // is to data owned by this column vector. If this column vector
+ // gets re-used, the whole thing is re-used together so there
+ // is no danger of a dangling reference.
+
+ // Only copy data values if entry is not null. The string value
+ // at position 0 is undefined if the position 0 value is null.
+ if (noNulls || (!noNulls && !isNull[0])) {
+
+ // loops start at position 1 because position 0 is already set
+ if (selectedInUse) {
+ for (int j = 1; j < size; j++) {
+ int i = sel[j];
+ this.setRef(i, vector[0], start[0], length[0]);
+ }
+ } else {
+ for (int i = 1; i < size; i++) {
+ this.setRef(i, vector[0], start[0], length[0]);
+ }
+ }
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ // Fill the all the vector entries with provided value
+ public void fill(byte[] value) {
+ noNulls = true;
+ isRepeating = true;
+ setRef(0, value, 0, value.length);
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java Thu Dec 12 19:01:09 2013
@@ -49,6 +49,11 @@ public abstract class ColumnVector {
* If so, vector[0] holds the repeating value.
*/
public boolean isRepeating;
+
+ // Variables to hold state from before flattening so it can be easily restored.
+ private boolean preFlattenIsRepeating;
+ private boolean preFlattenNoNulls;
+
public abstract Writable getWritableObject(int index);
/**
@@ -76,5 +81,66 @@ public abstract class ColumnVector {
noNulls = true;
isRepeating = false;
}
+
+ abstract public void flatten(boolean selectedInUse, int[] sel, int size);
+
+ // Simplify vector by brute-force flattening noNulls if isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flattenRepeatingNulls(boolean selectedInUse, int[] sel, int size) {
+
+ boolean nullFillValue;
+
+ if (noNulls) {
+ nullFillValue = false;
+ } else {
+ nullFillValue = isNull[0];
+ }
+
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ isNull[i] = nullFillValue;
+ }
+ } else {
+ Arrays.fill(isNull, 0, size, nullFillValue);
+ }
+
+ // all nulls are now explicit
+ noNulls = false;
+ }
+
+ public void flattenNoNulls(boolean selectedInUse, int[] sel, int size) {
+ if (noNulls) {
+ noNulls = false;
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ isNull[i] = false;
+ }
+ } else {
+ Arrays.fill(isNull, 0, size, false);
+ }
+ }
+ }
+
+ /**
+ * Restore the state of isRepeating and noNulls to what it was
+ * before flattening. This must only be called just after flattening
+ * and then evaluating a VectorExpression on the column vector.
+ * It is an optimization that allows other operations on the same
+ * column to continue to benefit from the isRepeating and noNulls
+ * indicators.
+ */
+ public void unFlatten() {
+ isRepeating = preFlattenIsRepeating;
+ noNulls = preFlattenNoNulls;
+ }
+
+ // Record repeating and no nulls state to be restored later.
+ protected void flattenPush() {
+ preFlattenIsRepeating = isRepeating;
+ preFlattenNoNulls = noNulls;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java Thu Dec 12 19:01:09 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
@@ -67,4 +69,76 @@ public class DoubleColumnVector extends
return writableObj;
}
}
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ System.arraycopy(vector, 0, output.vector, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Fill the column vector with the provided value
+ public void fill(double value) {
+ noNulls = true;
+ isRepeating = true;
+ vector[0] = value;
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ double repeatVal = vector[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ vector[i] = repeatVal;
+ }
+ } else {
+ Arrays.fill(vector, 0, size, repeatVal);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java Thu Dec 12 19:01:09 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
+
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
@@ -67,4 +69,120 @@ public class LongColumnVector extends Co
return writableObj;
}
}
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, LongColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ System.arraycopy(vector, 0, output.vector, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0]; // automatic conversion to double is done here
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ System.arraycopy(vector, 0, output.vector, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Fill the column vector with the provided value
+ public void fill(long value) {
+ noNulls = true;
+ isRepeating = true;
+ vector[0] = value;
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatVal = vector[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ vector[i] = repeatVal;
+ }
+ } else {
+ Arrays.fill(vector, 0, size, repeatVal);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringColumn.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringColumn.java?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringColumn.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringColumn.java Thu Dec 12 19:01:09 2013
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second and third are string columns or string expression results.
+ */
+public class IfExprStringColumnStringColumn extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg2Column, arg3Column;
+ private int outputColumn;
+
+ public IfExprStringColumnStringColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
+ }
+
+ public IfExprStringColumnStringColumn() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ BytesColumnVector arg2ColVector = (BytesColumnVector) batch.cols[arg2Column];
+ BytesColumnVector arg3ColVector = (BytesColumnVector) batch.cols[arg3Column];
+ BytesColumnVector outputColVector = (BytesColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls;
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.initBuffer();
+
+ /* All the code paths below propagate nulls even if neither arg2 nor arg3
+ * have nulls. This is to reduce the number of code paths and shorten the
+ * code, at the expense of maybe doing unnecessary work if neither input
+ * has nulls. This could be improved in the future by expanding the number
+ * of code paths.
+ */
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // extend any repeating values and noNulls indicator in the inputs
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+ arg2ColVector.unFlatten();
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "String";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public int getArg2Column() {
+ return arg2Column;
+ }
+
+ public void setArg2Column(int colNum) {
+ this.arg2Column = colNum;
+ }
+
+ public int getArg3Column() {
+ return arg3Column;
+ }
+
+ public void setArg3Column(int colNum) {
+ this.arg3Column = colNum;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringScalar.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringScalar.java?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringScalar.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringColumnStringScalar.java Thu Dec 12 19:01:09 2013
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a string column expression.
+ * The third is a string scalar.
+ */
+public class IfExprStringColumnStringScalar extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg2Column;
+ private byte[] arg3Scalar;
+ private int outputColumn;
+
+ public IfExprStringColumnStringScalar(int arg1Column, int arg2Column, byte[] arg3Scalar, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Column = arg2Column;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
+ }
+
+ public IfExprStringColumnStringScalar() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ BytesColumnVector arg2ColVector = (BytesColumnVector) batch.cols[arg2Column];
+ BytesColumnVector outputColVector = (BytesColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg2ColVector.noNulls;
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.initBuffer();
+
+ /* All the code paths below propagate nulls even if arg2 has no nulls.
+ * This is to reduce the number of code paths and shorten the
+ * code, at the expense of maybe doing unnecessary work if neither input
+ * has nulls. This could be improved in the future by expanding the number
+ * of code paths.
+ */
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ return;
+ }
+
+ // extend any repeating values and noNulls indicator in the inputs
+ arg2ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg3Scalar.length);
+ }
+ outputIsNull[i] = (vector1[i] == 1 ? arg2ColVector.isNull[i] : false);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg3Scalar.length);
+ }
+ outputIsNull[i] = (vector1[i] == 1 ? arg2ColVector.isNull[i] : false);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg3Scalar.length);
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ if (!arg2ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg2ColVector.vector[i], arg2ColVector.start[i], arg2ColVector.length[i]);
+ }
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg3Scalar.length);
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ arg2ColVector.isNull[i] : false);
+ }
+ }
+ }
+
+ // restore state of repeating and non nulls indicators
+ arg2ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "String";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public int getArg2Column() {
+ return arg2Column;
+ }
+
+ public void setArg2Column(int colNum) {
+ this.arg2Column = colNum;
+ }
+
+ public byte[] getArg3Scalar() {
+ return arg3Scalar;
+ }
+
+ public void setArg3Scalar(byte[] value) {
+ this.arg3Scalar = value;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringColumn.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringColumn.java?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringColumn.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringColumn.java Thu Dec 12 19:01:09 2013
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a string scalar.
+ * The third is a string column or non-constant expression result.
+ */
+public class IfExprStringScalarStringColumn extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column, arg3Column;
+ private byte[] arg2Scalar;
+ private int outputColumn;
+
+ public IfExprStringScalarStringColumn(int arg1Column, byte[] arg2Scalar, int arg3Column, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Column = arg3Column;
+ this.outputColumn = outputColumn;
+ }
+
+ public IfExprStringScalarStringColumn() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ BytesColumnVector arg3ColVector = (BytesColumnVector) batch.cols[arg3Column];
+ BytesColumnVector outputColVector = (BytesColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ boolean[] outputIsNull = outputColVector.isNull;
+ outputColVector.noNulls = arg3ColVector.noNulls;
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.initBuffer();
+
+ /* All the code paths below propagate nulls even arg3 has no
+ * nulls. This is to reduce the number of code paths and shorten the
+ * code, at the expense of maybe doing unnecessary work if neither input
+ * has nulls. This could be improved in the future by expanding the number
+ * of code paths.
+ */
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector);
+ }
+ return;
+ }
+
+ // extend any repeating values and noNulls indicator in the input
+ arg3ColVector.flatten(batch.selectedInUse, sel, n);
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (vector1[i] == 1 ? false : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (vector1[i] == 1 ? false : arg3ColVector.isNull[i]);
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ if (!arg3ColVector.isNull[i]) {
+ outputColVector.setVal(
+ i, arg3ColVector.vector[i], arg3ColVector.start[i], arg3ColVector.length[i]);
+ }
+ }
+ outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ?
+ false : arg3ColVector.isNull[i]);
+ }
+ }
+ }
+
+ // restore state of repeating and non nulls indicators
+ arg3ColVector.unFlatten();
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "String";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public byte[] getArg2Scalar() {
+ return arg2Scalar;
+ }
+
+ public void setArg2Scalar(byte[] value) {
+ this.arg2Scalar = value;
+ }
+
+ public int getArg3Column() {
+ return arg3Column;
+ }
+
+ public void setArg3Column(int colNum) {
+ this.arg3Column = colNum;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java?rev=1550488&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprStringScalarStringScalar.java Thu Dec 12 19:01:09 2013
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+
+/**
+ * Compute IF(expr1, expr2, expr3) for 3 input column expressions.
+ * The first is always a boolean (LongColumnVector).
+ * The second is a string scalar.
+ * The third is a string scalar.
+ */
+public class IfExprStringScalarStringScalar extends VectorExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int arg1Column;
+ private byte[] arg2Scalar;
+ private byte[] arg3Scalar;
+ private int outputColumn;
+
+ public IfExprStringScalarStringScalar(
+ int arg1Column, byte[] arg2Scalar, byte[] arg3Scalar, int outputColumn) {
+ this.arg1Column = arg1Column;
+ this.arg2Scalar = arg2Scalar;
+ this.arg3Scalar = arg3Scalar;
+ this.outputColumn = outputColumn;
+ }
+
+ public IfExprStringScalarStringScalar() {
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column];
+ BytesColumnVector outputColVector = (BytesColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ outputColVector.noNulls = true; // output must be a scalar and neither one is null
+ outputColVector.isRepeating = false; // may override later
+ int n = batch.size;
+ long[] vector1 = arg1ColVector.vector;
+
+ // return immediately if batch is empty
+ if (n == 0) {
+ return;
+ }
+
+ outputColVector.initBuffer();
+
+ if (arg1ColVector.isRepeating) {
+ if (vector1[0] == 1) {
+ outputColVector.fill(arg2Scalar);
+ } else {
+ outputColVector.fill(arg3Scalar);
+ }
+ return;
+ }
+
+ if (arg1ColVector.noNulls) {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg2Scalar.length);
+ }
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg2Scalar.length);
+ }
+ }
+ }
+ } else /* there are nulls */ {
+ if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg2Scalar.length);
+ }
+ }
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (!arg1ColVector.isNull[i] && vector1[i] == 1) {
+ outputColVector.setRef(i, arg2Scalar, 0, arg2Scalar.length);
+ } else {
+ outputColVector.setRef(i, arg3Scalar, 0, arg2Scalar.length);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "String";
+ }
+
+ public int getArg1Column() {
+ return arg1Column;
+ }
+
+ public void setArg1Column(int colNum) {
+ this.arg1Column = colNum;
+ }
+
+ public byte[] getArg2Scalar() {
+ return arg2Scalar;
+ }
+
+ public void setArg2Scalar(byte[] value) {
+ this.arg2Scalar = value;
+ }
+
+ public byte[] getArg3Scalar() {
+ return arg3Scalar;
+ }
+
+ public void setArg3Scalar(byte[] value) {
+ this.arg3Scalar = value;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ return (new VectorExpressionDescriptor.Builder())
+ .setMode(
+ VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(3)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.getType("long"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"),
+ VectorExpressionDescriptor.ArgumentType.getType("string"))
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR,
+ VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Thu Dec 12 19:01:09 2013
@@ -126,6 +126,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCeil;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcat;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFloor;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLower;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
@@ -261,6 +262,9 @@ public class Vectorizer implements Physi
supportedGenericUDFs.add(UDFToString.class);
supportedGenericUDFs.add(GenericUDFTimestamp.class);
+ // For conditional expressions
+ supportedGenericUDFs.add(GenericUDFIf.class);
+
supportedAggregationUdfs.add("min");
supportedAggregationUdfs.add("max");
supportedAggregationUdfs.add("count");
@@ -347,17 +351,17 @@ public class Vectorizer implements Physi
topNodes.addAll(mapWork.getAliasToWork().values());
HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutput);
-
+
Map<String, Map<Integer, String>> columnVectorTypes = vnp.getScratchColumnVectorTypes();
mapWork.setScratchColumnVectorTypes(columnVectorTypes);
Map<String, Map<String, Integer>> columnMap = vnp.getScratchColumnMap();
mapWork.setScratchColumnMap(columnMap);
-
+
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("vectorTypes: %s", columnVectorTypes.toString()));
LOG.debug(String.format("columnMap: %s", columnMap.toString()));
}
-
+
return;
}
}
@@ -426,9 +430,9 @@ public class Vectorizer implements Physi
Object... nodeOutputs) throws SemanticException {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
-
- VectorizationContext vContext = null;
-
+
+ VectorizationContext vContext = null;
+
if (op instanceof TableScanOperator) {
vContext = getVectorizationContext(op, physicalContext);
for (String onefile : mWork.getPathToAliases().keySet()) {
@@ -458,9 +462,9 @@ public class Vectorizer implements Physi
--i;
}
}
-
+
assert vContext != null;
-
+
if (op.getType().equals(OperatorType.REDUCESINK) &&
op.getParentOperators().get(0).getType().equals(OperatorType.GROUPBY)) {
// No need to vectorize
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java?rev=1550488&r1=1550487&r2=1550488&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java Thu Dec 12 19:01:09 2013
@@ -21,11 +21,30 @@ package org.apache.hadoop.hive.ql.udf.ge
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnLongScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnDoubleScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleColumnLongScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongColumnDoubleScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarLongColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarDoubleColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarLongScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarDoubleScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprDoubleScalarLongScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.IfExprLongScalarDoubleScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringColumnStringScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarStringScalar;
/**
* IF(expr1,expr2,expr3) <br>
@@ -33,6 +52,17 @@ import org.apache.hadoop.hive.serde2.obj
* otherwise it returns expr3. IF() returns a numeric or string value, depending
* on the context in which it is used.
*/
+@VectorizedExpressions({
+ IfExprLongColumnLongColumn.class, IfExprDoubleColumnDoubleColumn.class,
+ IfExprLongColumnLongScalar.class, IfExprDoubleColumnDoubleScalar.class,
+ IfExprLongColumnDoubleScalar.class, IfExprDoubleColumnLongScalar.class,
+ IfExprLongScalarLongColumn.class, IfExprDoubleScalarDoubleColumn.class,
+ IfExprLongScalarDoubleColumn.class, IfExprDoubleScalarLongColumn.class,
+ IfExprLongScalarLongScalar.class, IfExprDoubleScalarDoubleScalar.class,
+ IfExprLongScalarDoubleScalar.class, IfExprDoubleScalarLongScalar.class,
+ IfExprStringColumnStringColumn.class, IfExprStringColumnStringScalar.class,
+ IfExprStringScalarStringColumn.class, IfExprStringScalarStringScalar.class
+})
public class GenericUDFIf extends GenericUDF {
private transient ObjectInspector[] argumentOIs;
private transient GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
@@ -94,5 +124,4 @@ public class GenericUDFIf extends Generi
sb.append(children[2]).append(")");
return sb.toString();
}
-
}