You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/29 17:12:40 UTC
svn commit: r1477136 [3/4] - in /hive/branches/vectorization/ql/src:
java/org/apache/hadoop/hive/ql/exec/vector/
java/org/apache/hadoop/hive/ql/exec/vector/expressions/
java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/
java/org/apache/...
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates
+ .VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* VectorUDAFVarSampDouble. Vectorized implementation for VARIANCE aggregates.
+*/
+@Description(name = "var_samp", value = "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, double)")
+public class VectorUDAFVarSampDouble extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+ double sum;
+ long count;
+ double variance;
+ boolean isNull;
+
+ public void init () {
+ isNull = false;
+ sum = 0;
+ count = 0;
+ variance = 0;
+ }
+ }
+
+ private VectorExpression inputExpression;
+ private LongWritable resultCount;
+ private DoubleWritable resultSum;
+ private DoubleWritable resultVariance;
+ private Object[] partialResult;
+
+ private ObjectInspector soi;
+
+
+ public VectorUDAFVarSampDouble(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ partialResult = new Object[3];
+ resultCount = new LongWritable();
+ resultSum = new DoubleWritable();
+ resultVariance = new DoubleWritable();
+ partialResult[0] = resultCount;
+ partialResult[1] = resultSum;
+ partialResult[2] = resultVariance;
+ initPartialResultInspector();
+ }
+
+ private void initPartialResultInspector () {
+ ArrayList<ObjectInspector> foi = new ArrayList<ObjectInspector>();
+ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+
+ ArrayList<String> fname = new ArrayList<String>();
+ fname.add("count");
+ fname.add("sum");
+ fname.add("variance");
+
+ soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
+ throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ DoubleColumnVector inputVector = (DoubleColumnVector)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ double[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
+ iterateRepeatingNoNulls(myagg, vector[0], batchSize);
+ }
+ }
+ else if (!unit.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, unit.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateRepeatingNoNulls(
+ Aggregation myagg,
+ double value,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ // TODO: conjure a formula w/o iterating
+ //
+
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // We pulled out i=0 so we can remove the count > 1 check in the loop
+ for (int i=1; i<batchSize; ++i) {
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ double[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ double value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ double[] vector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ double value = vector[selected[0]];
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove the count > 1 check in the loop
+ //
+ for (int i=1; i< batchSize; ++i) {
+ value = vector[selected[i]];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ double[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ double value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ double[] vector,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ double value = vector[0];
+ myagg.sum += value;
+ myagg.count += 1;
+
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove count > 1 check
+ for (int i=1; i<batchSize; ++i) {
+ value = vector[i];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(
+ AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ assert(0 < myagg.count);
+ resultCount.set (myagg.count);
+ resultSum.set (myagg.sum);
+ resultVariance.set (myagg.variance);
+ return partialResult;
+ }
+ }
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return soi;
+ }
+}
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates
+ .VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* VectorUDAFVarSampLong. Vectorized implementation for VARIANCE aggregates.
+*/
+@Description(name = "var_samp", value = "_FUNC_(x) - Returns the sample variance of a set of numbers (vectorized, long)")
+public class VectorUDAFVarSampLong extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+ double sum;
+ long count;
+ double variance;
+ boolean isNull;
+
+ public void init () {
+ isNull = false;
+ sum = 0;
+ count = 0;
+ variance = 0;
+ }
+ }
+
+ private VectorExpression inputExpression;
+ private LongWritable resultCount;
+ private DoubleWritable resultSum;
+ private DoubleWritable resultVariance;
+ private Object[] partialResult;
+
+ private ObjectInspector soi;
+
+
+ public VectorUDAFVarSampLong(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ partialResult = new Object[3];
+ resultCount = new LongWritable();
+ resultSum = new DoubleWritable();
+ resultVariance = new DoubleWritable();
+ partialResult[0] = resultCount;
+ partialResult[1] = resultSum;
+ partialResult[2] = resultVariance;
+ initPartialResultInspector();
+ }
+
+ private void initPartialResultInspector () {
+ ArrayList<ObjectInspector> foi = new ArrayList<ObjectInspector>();
+ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+
+ ArrayList<String> fname = new ArrayList<String>();
+ fname.add("count");
+ fname.add("sum");
+ fname.add("variance");
+
+ soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
+ throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ LongColumnVector inputVector = (LongColumnVector)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ long[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
+ iterateRepeatingNoNulls(myagg, vector[0], batchSize);
+ }
+ }
+ else if (!unit.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, unit.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateRepeatingNoNulls(
+ Aggregation myagg,
+ long value,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ // TODO: conjure a formula w/o iterating
+ //
+
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // We pulled out i=0 so we can remove the count > 1 check in the loop
+ for (int i=1; i<batchSize; ++i) {
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ long value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ long value = vector[selected[0]];
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove the count > 1 check in the loop
+ //
+ for (int i=1; i< batchSize; ++i) {
+ value = vector[selected[i]];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ long value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ long value = vector[0];
+ myagg.sum += value;
+ myagg.count += 1;
+
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove count > 1 check
+ for (int i=1; i<batchSize; ++i) {
+ value = vector[i];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(
+ AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ assert(0 < myagg.count);
+ resultCount.set (myagg.count);
+ resultSum.set (myagg.sum);
+ resultVariance.set (myagg.variance);
+ return partialResult;
+ }
+ }
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return soi;
+ }
+}
+
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java?rev=1477136&r1=1477135&r2=1477136&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java Mon Apr 29 15:12:39 2013
@@ -258,8 +258,16 @@ public class CodeGen {
generateFilterColumnCompareColumn(tdesc);
} else if (tdesc[0].equals("ColumnArithmeticColumn")) {
generateColumnArithmeticColumn(tdesc);
- } else if (tdesc[0].equals("ColumnUnaryMinus")) {
+ } else if (tdesc[0].equals("ColumnUnaryMinus")) {
generateColumnUnaryMinus(tdesc);
+ } else if (tdesc[0].equals("VectorUDAFCount")) {
+ generateVectorUDAFCount(tdesc);
+ } else if (tdesc[0].equals("VectorUDAFSum")) {
+ generateVectorUDAFSum(tdesc);
+ } else if (tdesc[0].equals("VectorUDAFAvg")) {
+ generateVectorUDAFAvg(tdesc);
+ } else if (tdesc[0].equals("VectorUDAFVar")) {
+ generateVectorUDAFVar(tdesc);
} else {
continue;
}
@@ -607,3 +615,4 @@ public class CodeGen {
throw new Exception("Unimplemented primitive output inspector: " + primitiveType);
}
}
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFCount.txt Mon Apr 29 15:12:39 2013
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.
+ VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* <ClassName>. Vectorized implementation for COUNT aggregates.
+*/
+@Description(name = "count", value = "_FUNC_(expr) - Returns the maximum value of expr (vectorized, type: <ValueType>)")
+public class <ClassName> extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static class Aggregation implements AggregationBuffer {
+ long value;
+ boolean isNull;
+ }
+
+ private VectorExpression inputExpression;
+ private LongWritable result;
+
+ public <ClassName>(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ result = new LongWritable(0);
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
+ throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ <InputColumnVectorType> inputVector = (<InputColumnVectorType>)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ if (myagg.isNull) {
+ myagg.value = 0;
+ myagg.isNull = false;
+ }
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
+ myagg.value += batchSize;
+ }
+ return;
+ }
+
+ if (inputVector.noNulls) {
+ myagg.value += batchSize;
+ return;
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, batchSize, inputVector.isNull);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ myagg.value += 1;
+ }
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i< batchSize; ++i) {
+ if (!isNull[i]) {
+ myagg.value += 1;
+ }
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ result.set (myagg.value);
+ return result;
+ }
+ }
+
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ }
+
+}
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFMinMax.txt Mon Apr 29 15:12:39 2013
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.
+ VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* <ClassName>. Vectorized implementation for MIN/MAX aggregates.
+*/
+@Description(name = "<DescriptionName>", value = "<DescriptionValue>")
+public class <ClassName> extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+ <ValueType> value;
+ boolean isNull;
+ }
+
+ private VectorExpression inputExpression;
+ private <OutputType> result;
+
+ public <ClassName>(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ result = new <OutputType>();
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit) throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ <InputColumnVectorType> inputVector = (<InputColumnVectorType>)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ <ValueType>[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if ((inputVector.noNulls || !inputVector.isNull[0]) &&
+ myagg.isNull || vector[0] < myagg.value) {
+ myagg.isNull = false;
+ myagg.value = vector[0];
+ }
+ return;
+ }
+
+ if (!unit.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, unit.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.isNull = false;
+ myagg.value = value;
+ }
+ else if (value <OperatorSymbol> myagg.value) {
+ myagg.value = value;
+ }
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ myagg.value = vector[selected[0]];
+ myagg.isNull = false;
+ }
+
+ for (int i=0; i< batchSize; ++i) {
+ <ValueType> value = vector[selected[i]];
+ if (value <OperatorSymbol> myagg.value) {
+ myagg.value = value;
+ }
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.value = value;
+ myagg.isNull = false;
+ }
+ else if (value <OperatorSymbol> myagg.value) {
+ myagg.value = value;
+ }
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize) {
+ if (myagg.isNull) {
+ myagg.value = vector[0];
+ myagg.isNull = false;
+ }
+
+ for (int i=0;i<batchSize;++i) {
+ <ValueType> value = vector[i];
+ if (value <OperatorSymbol> myagg.value) {
+ myagg.value = value;
+ }
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(
+ AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ result.set(myagg.value);
+ return result;
+ }
+ }
+
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return <OutputTypeInspector>;
+ }
+}
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFSum.txt Mon Apr 29 15:12:39 2013
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.
+ VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* <ClassName>. Vectorized implementation for SUM aggregates.
+*/
+@Description(name = "sum", value = "_FUNC_(expr) - Returns the sum value of expr (vectorized, type: <ValueType>)")
+public class <ClassName> extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+ <ValueType> sum;
+ boolean isNull;
+ }
+
+ VectorExpression inputExpression;
+ private <OutputType> result;
+
+ public <ClassName>(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ result = new <OutputType>();
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
+ throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ <InputColumnVectorType> inputVector = (<InputColumnVectorType>)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ <ValueType>[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
+ if (myagg.isNull) {
+ myagg.isNull = false;
+ myagg.sum = 0;
+ }
+ myagg.sum += vector[0]*batchSize;
+ }
+ return;
+ }
+
+ if (!unit.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, unit.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.isNull = false;
+ myagg.sum = 0;
+ }
+ myagg.sum += value;
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ myagg.sum = 0;
+ myagg.isNull = false;
+ }
+
+ for (int i=0; i< batchSize; ++i) {
+ <ValueType> value = vector[selected[i]];
+ myagg.sum += value;
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.sum = 0;
+ myagg.isNull = false;
+ }
+ myagg.sum += value;
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize) {
+ if (myagg.isNull) {
+ myagg.sum = 0;
+ myagg.isNull = false;
+ }
+
+ for (int i=0;i<batchSize;++i) {
+ <ValueType> value = vector[i];
+ myagg.sum += value;
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ result.set(myagg.sum);
+ return result;
+ }
+ }
+
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return <OutputTypeInspector>;
+ }
+}
+
Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt Mon Apr 29 15:12:39 2013
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates
+ .VectorAggregateExpression.AggregationBuffer;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+/**
+* <ClassName>. Vectorized implementation for VARIANCE aggregates.
+*/
+@Description(name = "<DescriptionName>", value = "<DescriptionValue>")
+public class <ClassName> extends VectorAggregateExpression {
+
+ /**
+ /* class for storing the current aggregate value.
+ */
+ static private final class Aggregation implements AggregationBuffer {
+ double sum;
+ long count;
+ double variance;
+ boolean isNull;
+
+ public void init () {
+ isNull = false;
+ sum = 0;
+ count = 0;
+ variance = 0;
+ }
+ }
+
+ private VectorExpression inputExpression;
+ private LongWritable resultCount;
+ private DoubleWritable resultSum;
+ private DoubleWritable resultVariance;
+ private Object[] partialResult;
+
+ private ObjectInspector soi;
+
+
+ public <ClassName>(VectorExpression inputExpression) {
+ super();
+ this.inputExpression = inputExpression;
+ partialResult = new Object[3];
+ resultCount = new LongWritable();
+ resultSum = new DoubleWritable();
+ resultVariance = new DoubleWritable();
+ partialResult[0] = resultCount;
+ partialResult[1] = resultSum;
+ partialResult[2] = resultVariance;
+ initPartialResultInspector();
+ }
+
+ private void initPartialResultInspector () {
+ ArrayList<ObjectInspector> foi = new ArrayList<ObjectInspector>();
+ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+ foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+
+ ArrayList<String> fname = new ArrayList<String>();
+ fname.add("count");
+ fname.add("sum");
+ fname.add("variance");
+
+ soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch unit)
+ throws HiveException {
+
+ inputExpression.evaluate(unit);
+
+ <InputColumnVectorType> inputVector = (<InputColumnVectorType>)unit.
+ cols[this.inputExpression.getOutputColumn()];
+
+ int batchSize = unit.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ <ValueType>[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
+ iterateRepeatingNoNulls(myagg, vector[0], batchSize);
+ }
+ }
+ else if (!unit.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ }
+ else if (!unit.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ }
+ else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, unit.selected);
+ }
+ else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, unit.selected);
+ }
+ }
+
+ private void iterateRepeatingNoNulls(
+ Aggregation myagg,
+ <ValueType> value,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ // TODO: conjure a formula w/o iterating
+ //
+
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // We pulled out i=0 so we can remove the count > 1 check in the loop
+ for (int i=1; i<batchSize; ++i) {
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ int[] selected) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ <ValueType> value = vector[selected[0]];
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove the count > 1 check in the loop
+ //
+ for (int i=1; i< batchSize; ++i) {
+ value = vector[selected[i]];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ <ValueType> value = vector[i];
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+ myagg.sum += value;
+ myagg.count += 1;
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize) {
+
+ if (myagg.isNull) {
+ myagg.init ();
+ }
+
+ <ValueType> value = vector[0];
+ myagg.sum += value;
+ myagg.count += 1;
+
+ if(myagg.count > 1) {
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+
+ // i=0 was pulled out to remove count > 1 check
+ for (int i=1; i<batchSize; ++i) {
+ value = vector[i];
+ myagg.sum += value;
+ myagg.count += 1;
+ double t = myagg.count*value - myagg.sum;
+ myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation();
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.isNull = true;
+ }
+
+ @Override
+ public Object evaluateOutput(
+ AggregationBuffer agg) throws HiveException {
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ return null;
+ }
+ else {
+ assert(0 < myagg.count);
+ resultCount.set (myagg.count);
+ resultSum.set (myagg.sum);
+ resultVariance.set (myagg.variance);
+ return partialResult;
+ }
+ }
+ @Override
+ public ObjectInspector getOutputObjectInspector() {
+ return soi;
+ }
+}
+
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,919 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.vector.util.FakeCaptureOutputOperator;
+import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromConcat;
+import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromIterables;
+import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromRepeats;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.LongWritable;
+import org.junit.Test;
+
+/**
+ * Unit test for the vectorized GROUP BY operator.
+ */
+public class TestVectorGroupByOperator {
+
+ private static ExprNodeDesc buildColumnDesc(
+ VectorizationContext ctx,
+ String column) {
+ return new ExprNodeColumnDesc(
+ TypeInfoFactory.longTypeInfo, column, "table", false);
+ }
+
+ private static AggregationDesc buildAggregationDesc(
+ VectorizationContext ctx,
+ String aggregate,
+ String column) {
+
+ ExprNodeDesc inputColumn = buildColumnDesc(ctx, column);
+
+ ArrayList<ExprNodeDesc> params = new ArrayList<ExprNodeDesc>();
+ params.add(inputColumn);
+
+ AggregationDesc agg = new AggregationDesc();
+ agg.setGenericUDAFName(aggregate);
+ agg.setParameters(params);
+
+ return agg;
+ }
+
+ private static GroupByDesc buildGroupByDesc(
+ VectorizationContext ctx,
+ String aggregate,
+ String column) {
+
+ AggregationDesc agg = buildAggregationDesc(ctx, aggregate, column);
+ ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
+ aggs.add(agg);
+
+ ArrayList<String> outputColumnNames = new ArrayList<String>();
+ outputColumnNames.add("_col0");
+
+ GroupByDesc desc = new GroupByDesc();
+ desc.setOutputColumnNames(outputColumnNames);
+ desc.setAggregators(aggs);
+
+ return desc;
+ }
+
+ @Test
+ public void testMinLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ 5L);
+ }
+
+ @Test
+ public void testMinLongEmpty() throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+ @Test
+ public void testMinLongNulls() throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{null}),
+ null);
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{null, null, null}),
+ null);
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{null,5L,7L,19L}),
+ 5L);
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{13L,null,7L,19L}),
+ 7L);
+ }
+
+ @Test
+ public void testMinLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "min",
+ 42L,
+ 4096,
+ 1024,
+ 42L);
+ }
+
+ @Test
+ public void testMinLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "min",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+
+ @Test
+ public void testMinLongNegative () throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,-19L}),
+ -19L);
+ }
+
+ @Test
+ public void testMinLongMinInt () throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,(long)Integer.MIN_VALUE,-19L}),
+ (long)Integer.MIN_VALUE);
+ }
+
+ @Test
+ public void testMinLongMinLong () throws HiveException {
+ testAggregateLongAggregate(
+ "min",
+ 2,
+ Arrays.asList(new Long[]{13L,5L, Long.MIN_VALUE, (long)Integer.MIN_VALUE}),
+ Long.MIN_VALUE);
+ }
+
+ @Test
+ public void testMaxLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "max",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ 19L);
+ }
+
+ @Test
+ public void testMaxLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "max",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+
+ @Test
+ public void testMaxLongNegative () throws HiveException {
+ testAggregateLongAggregate(
+ "max",
+ 2,
+ Arrays.asList(new Long[]{-13L,-5L,-7L,-19L}),
+ -5L);
+ }
+
+ @Test
+ public void testMaxLongMaxInt () throws HiveException {
+ testAggregateLongAggregate(
+ "max",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,(long)Integer.MAX_VALUE}),
+ (long)Integer.MAX_VALUE);
+ }
+
+ @Test
+ public void testMaxLongMaxLong () throws HiveException {
+ testAggregateLongAggregate(
+ "max",
+ 2,
+ Arrays.asList(new Long[]{13L,Long.MAX_VALUE - 1L,Long.MAX_VALUE,(long)Integer.MAX_VALUE}),
+ Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testMaxLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "max",
+ 42L,
+ 4096,
+ 1024,
+ 42L);
+ }
+
+ @Test
+ public void testMaxLongNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "max",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMinLongConcatRepeat () throws HiveException {
+ testAggregateLongIterable ("min",
+ new FakeVectorRowBatchFromConcat(
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2),
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {7L}, 15, 2),
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2)),
+ 7L);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMinLongRepeatConcatValues () throws HiveException {
+ testAggregateLongIterable ("min",
+ new FakeVectorRowBatchFromConcat(
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2),
+ new FakeVectorRowBatchFromIterables(
+ 3,
+ Arrays.asList(new Long[]{13L, 7L, 23L, 29L}))),
+ 7L);
+ }
+
+ @Test
+ public void testCountLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ 4L);
+ }
+
+ @Test
+ public void testCountLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+ @Test
+ public void testCountLongNulls () throws HiveException {
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{null}),
+ 0L);
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{null, null, null}),
+ 0L);
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{null,5L,7L,19L}),
+ 3L);
+ testAggregateLongAggregate(
+ "count",
+ 2,
+ Arrays.asList(new Long[]{13L,null,7L,19L}),
+ 3L);
+ }
+
+
+ @Test
+ public void testCountLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "count",
+ 42L,
+ 4096,
+ 1024,
+ 4096L);
+ }
+
+ @Test
+ public void testCountLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "count",
+ null,
+ 4096,
+ 1024,
+ 0L);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testCountLongRepeatConcatValues () throws HiveException {
+ testAggregateLongIterable ("count",
+ new FakeVectorRowBatchFromConcat(
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2),
+ new FakeVectorRowBatchFromIterables(
+ 3,
+ Arrays.asList(new Long[]{13L, 7L, 23L, 29L}))),
+ 14L);
+ }
+
+ @Test
+ public void testSumLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ 13L + 5L + 7L + 19L);
+ }
+
+ @Test
+ public void testSumLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+ @Test
+ public void testSumLongNulls () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{null}),
+ null);
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{null, null, null}),
+ null);
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{null,5L,7L,19L}),
+ 5L + 7L + 19L);
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{13L,null,7L,19L}),
+ 13L + 7L + 19L);
+ }
+
+ @Test
+ public void testSumLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "sum",
+ 42L,
+ 4096,
+ 1024,
+ 4096L * 42L);
+ }
+
+ @Test
+ public void testSumLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "sum",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testSumLongRepeatConcatValues () throws HiveException {
+ testAggregateLongIterable ("sum",
+ new FakeVectorRowBatchFromConcat(
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2),
+ new FakeVectorRowBatchFromIterables(
+ 3,
+ Arrays.asList(new Long[]{13L, 7L, 23L, 29L}))),
+ 19L*10L + 13L + 7L + 23L +29L);
+ }
+
+ @Test
+ public void testSumLongZero () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{-(long)Integer.MAX_VALUE, (long)Integer.MAX_VALUE}),
+ 0L);
+ }
+
+ @Test
+ public void testSumLong2MaxInt () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{(long)Integer.MAX_VALUE, (long)Integer.MAX_VALUE}),
+ 4294967294L);
+ }
+
+ @Test
+ public void testSumLong2MinInt () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{(long)Integer.MIN_VALUE, (long)Integer.MIN_VALUE}),
+ -4294967296L);
+ }
+
+ @Test
+ public void testSumLong2MaxLong () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{Long.MAX_VALUE, Long.MAX_VALUE}),
+ -2L); // silent overflow
+ }
+
+ @Test
+ public void testSumLong2MinLong () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{Long.MIN_VALUE, Long.MIN_VALUE}),
+ 0L); // silent overflow
+ }
+
+ @Test
+ public void testSumLongMinMaxLong () throws HiveException {
+ testAggregateLongAggregate(
+ "sum",
+ 2,
+ Arrays.asList(new Long[]{Long.MAX_VALUE, Long.MIN_VALUE}),
+ -1L);
+ }
+
+ @Test
+ public void testAvgLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ (double) (13L + 5L + 7L + 19L) / (double) 4L);
+ }
+
+ @Test
+ public void testAvgLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+ @Test
+ public void testAvgLongNulls () throws HiveException {
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{null}),
+ null);
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{null, null, null}),
+ null);
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{null,5L,7L,19L}),
+ (double) (5L + 7L + 19L) / (double) 3L);
+ testAggregateLongAggregate(
+ "avg",
+ 2,
+ Arrays.asList(new Long[]{13L,null,7L,19L}),
+ (double) (13L + + 7L + 19L) / (double) 3L);
+ }
+
+
+ @Test
+ public void testAvgLongRepeat () throws HiveException
+ {
+ testAggregateLongRepeats (
+ "avg",
+ 42L,
+ 4096,
+ 1024,
+ (double)42);
+ }
+
+ @Test
+ public void testAvgLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "avg",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testAvgLongRepeatConcatValues () throws HiveException {
+ testAggregateLongIterable ("avg",
+ new FakeVectorRowBatchFromConcat(
+ new FakeVectorRowBatchFromRepeats(
+ new Long[] {19L}, 10, 2),
+ new FakeVectorRowBatchFromIterables(
+ 3,
+ Arrays.asList(new Long[]{13L, 7L, 23L, 29L}))),
+ (double) (19L*10L + 13L + 7L + 23L +29L) / (double) 14 );
+ }
+
+ @Test
+ public void testVarianceLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ (double) 30L);
+ }
+
+ @Test
+ public void testVarianceLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+ @Test
+ public void testVarianceLongSingle () throws HiveException {
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{97L}),
+ (double)0.0);
+ }
+
+ @Test
+ public void testVarianceLongNulls () throws HiveException {
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{null}),
+ null);
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{null, null, null}),
+ null);
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{null,13L, 5L,7L,19L}),
+ (double) 30.0);
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{13L,null,5L, 7L,19L}),
+ (double) 30.0);
+ testAggregateLongAggregate(
+ "variance",
+ 2,
+ Arrays.asList(new Long[]{null,null,null,19L}),
+ (double) 0);
+ }
+
+ @Test
+ public void testVarPopLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "var_pop",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+ @Test
+ public void testVarPopLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "var_pop",
+ 42L,
+ 4096,
+ 1024,
+ (double)0);
+ }
+
+ @Test
+ public void testVarSampLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "var_samp",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ (double) 40L);
+ }
+
+ @Test
+ public void testVarSampLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "var_samp",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+
+ @Test
+ public void testVarSampLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "var_samp",
+ 42L,
+ 4096,
+ 1024,
+ (double)0);
+ }
+
+ @Test
+ public void testStdLongSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "std",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ (double) Math.sqrt(30));
+ }
+
+ @Test
+ public void testStdLongEmpty () throws HiveException {
+ testAggregateLongAggregate(
+ "std",
+ 2,
+ Arrays.asList(new Long[]{}),
+ null);
+ }
+
+
+ @Test
+ public void testStdDevLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "stddev",
+ 42L,
+ 4096,
+ 1024,
+ (double)0);
+ }
+
+ @Test
+ public void testStdDevLongRepeatNulls () throws HiveException {
+ testAggregateLongRepeats (
+ "stddev",
+ null,
+ 4096,
+ 1024,
+ null);
+ }
+
+
+ @Test
+ public void testStdDevSampSimple () throws HiveException {
+ testAggregateLongAggregate(
+ "stddev_samp",
+ 2,
+ Arrays.asList(new Long[]{13L,5L,7L,19L}),
+ (double) Math.sqrt(40));
+ }
+
+ @Test
+ public void testStdDevSampLongRepeat () throws HiveException {
+ testAggregateLongRepeats (
+ "stddev_samp",
+ 42L,
+ 3,
+ 1024,
+ (double)0);
+ }
+
+
+ public void testAggregateLongRepeats (
+ String aggregateName,
+ Long value,
+ int repeat,
+ int batchSize,
+ Object expected) throws HiveException {
+ FakeVectorRowBatchFromRepeats fdr = new FakeVectorRowBatchFromRepeats(
+ new Long[] {value}, repeat, batchSize);
+ testAggregateLongIterable (aggregateName, fdr, expected);
+ }
+
+ public void testAggregateLongAggregate (
+ String aggregateName,
+ int batchSize,
+ Iterable<Long> values,
+ Object expected) throws HiveException {
+
+ @SuppressWarnings("unchecked")
+ FakeVectorRowBatchFromIterables fdr = new FakeVectorRowBatchFromIterables(batchSize, values);
+ testAggregateLongIterable (aggregateName, fdr, expected);
+ }
+
+ public static interface Validator {
+ void validate (Object expected, Object result);
+ };
+
+ public static class ValueValidator implements Validator {
+ @Override
+ public void validate(Object expected, Object result) {
+
+ assertEquals(true, result instanceof Object[]);
+ Object[] arr = (Object[]) result;
+ assertEquals (1, arr.length);
+
+ if (expected == null) {
+ assertNull (arr[0]);
+ } else {
+ assertEquals (true, arr[0] instanceof LongWritable);
+ LongWritable lw = (LongWritable) arr[0];
+ assertEquals ((Long) expected, (Long) lw.get());
+ }
+ }
+ }
+
+ public static class AvgValidator implements Validator {
+
+ @Override
+ public void validate(Object expected, Object result) {
+ Object[] arr = (Object[]) result;
+ assertEquals (1, arr.length);
+
+ if (expected == null) {
+ assertNull (arr[0]);
+ } else {
+ assertEquals (true, arr[0] instanceof Object[]);
+ Object[] vals = (Object[]) arr[0];
+ assertEquals (2, vals.length);
+
+ assertEquals (true, vals[0] instanceof LongWritable);
+ assertEquals (true, vals[1] instanceof DoubleWritable);
+ LongWritable lw = (LongWritable) vals[0];
+ DoubleWritable dw = (DoubleWritable) vals[1];
+ assertFalse (lw.get() == 0L);
+ assertEquals ((Double) expected, (Double) (dw.get() / lw.get()));
+ }
+ }
+
+ }
+
+ public abstract static class BaseVarianceValidator implements Validator {
+
+ abstract void validateVariance (
+ double expected, long cnt, double sum, double variance);
+
+ @Override
+ public void validate(Object expected, Object result) {
+ Object[] arr = (Object[]) result;
+ assertEquals (1, arr.length);
+
+ if (expected == null) {
+ assertNull (arr[0]);
+ } else {
+ assertEquals (true, arr[0] instanceof Object[]);
+ Object[] vals = (Object[]) arr[0];
+ assertEquals (3, vals.length);
+
+ assertEquals (true, vals[0] instanceof LongWritable);
+ assertEquals (true, vals[1] instanceof DoubleWritable);
+ assertEquals (true, vals[2] instanceof DoubleWritable);
+ LongWritable cnt = (LongWritable) vals[0];
+ DoubleWritable sum = (DoubleWritable) vals[1];
+ DoubleWritable var = (DoubleWritable) vals[2];
+ assertTrue (1 <= cnt.get());
+ validateVariance ((Double) expected, cnt.get(), sum.get(), var.get());
+ }
+ }
+ }
+
+ public static class VarianceValidator extends BaseVarianceValidator {
+
+ @Override
+ void validateVariance(double expected, long cnt, double sum, double variance) {
+ assertEquals (expected, variance /cnt, 0.0);
+ }
+ }
+
+ public static class VarianceSampValidator extends BaseVarianceValidator {
+
+ @Override
+ void validateVariance(double expected, long cnt, double sum, double variance) {
+ assertEquals (expected, variance /(cnt-1), 0.0);
+ }
+ }
+
+ public static class StdValidator extends BaseVarianceValidator {
+
+ @Override
+ void validateVariance(double expected, long cnt, double sum, double variance) {
+ assertEquals (expected, Math.sqrt(variance / cnt), 0.0);
+ }
+ }
+
+ public static class StdSampValidator extends BaseVarianceValidator {
+
+ @Override
+ void validateVariance(double expected, long cnt, double sum, double variance) {
+ assertEquals (expected, Math.sqrt(variance / (cnt-1)), 0.0);
+ }
+ }
+
+ private static Object[][] validators = {
+ {"count", ValueValidator.class},
+ {"min", ValueValidator.class},
+ {"max", ValueValidator.class},
+ {"sum", ValueValidator.class},
+ {"avg", AvgValidator.class},
+ {"variance", VarianceValidator.class},
+ {"var_pop", VarianceValidator.class},
+ {"var_samp", VarianceSampValidator.class},
+ {"std", StdValidator.class},
+ {"stddev", StdValidator.class},
+ {"stddev_samp", StdSampValidator.class},
+ };
+
+ public static Validator getValidator(String aggregate) throws HiveException {
+ try
+ {
+ for (Object[] v: validators) {
+ if (aggregate.equalsIgnoreCase((String) v[0])) {
+ @SuppressWarnings("unchecked")
+ Class<? extends Validator> c = (Class<? extends Validator>) v[1];
+ Constructor<? extends Validator> ctr = c.getConstructor();
+ return ctr.newInstance();
+ }
+ }
+ }catch(Exception e) {
+ throw new HiveException(e);
+ }
+ throw new HiveException("Missing validator for aggregate: " + aggregate);
+ }
+
+ public void testAggregateLongIterable (
+ String aggregateName,
+ Iterable<VectorizedRowBatch> data,
+ Object expected) throws HiveException {
+ Map<String, Integer> mapColumnNames = new HashMap<String, Integer>();
+ mapColumnNames.put("A", 0);
+ VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1);
+
+ GroupByDesc desc = buildGroupByDesc (ctx, aggregateName, "A");
+
+ VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
+
+ FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
+ vgo.initialize(null, null);
+
+ for (VectorizedRowBatch unit: data) {
+ vgo.process(unit, 0);
+ }
+ vgo.close(false);
+
+ List<Object> outBatchList = out.getCapturedRows();
+ assertNotNull(outBatchList);
+ assertEquals(1, outBatchList.size());
+
+ Object result = outBatchList.get(0);
+
+ Validator validator = getValidator(aggregateName);
+ validator.validate(expected, result);
+ }
+}
+
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputDesc.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputDesc.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputDesc.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc;
+
+/**
+ * Descriptor used for the FakeCaptureOutputOperator.
+ * Used in unit test only.
+ *
+ */
+public class FakeCaptureOutputDesc extends AbstractOperatorDesc {
+ private static final long serialVersionUID = 1L;
+}
+
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+
+/**
+ * Operator that captures output emitted by parent.
+ * Used in unit test only.
+ */
+public class FakeCaptureOutputOperator extends Operator<FakeCaptureOutputDesc>
+ implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private transient List<Object> rows;
+
+ public static FakeCaptureOutputOperator addCaptureOutputChild(
+ Operator<? extends OperatorDesc> op) {
+ FakeCaptureOutputOperator out = new FakeCaptureOutputOperator();
+ List<Operator<? extends OperatorDesc>> listParents =
+ new ArrayList<Operator<? extends OperatorDesc>>(1);
+ listParents.add(op);
+ out.setParentOperators(listParents);
+ List<Operator<? extends OperatorDesc>> listChildren =
+ new ArrayList<Operator<? extends OperatorDesc>>(1);
+ listChildren.add(out);
+ op.setChildOperators(listChildren);
+ return out;
+ }
+
+ public List<Object> getCapturedRows() {
+ return rows;
+ }
+
+ @Override
+ public void initializeOp(Configuration conf) throws HiveException {
+ rows = new ArrayList<Object>();
+ }
+
+ @Override
+ public void processOp(Object row, int tag) throws HiveException {
+ rows.add(row);
+ }
+
+ @Override
+ public OperatorType getType() {
+ return null;
+ }
+
+}
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java?rev=1477136&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java Mon Apr 29 15:12:39 2013
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+
+/**
+ * Vectorized data source operator for testing.
+ * Used in unit test only.
+ *
+ */
+public class FakeVectorDataSourceOperator extends Operator<FakeVectorDataSourceOperatorDesc>
+ implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private transient Iterable<VectorizedRowBatch> source;
+
+ public static FakeVectorDataSourceOperator addFakeVectorDataSourceParent(
+ Iterable<VectorizedRowBatch> source,
+ Operator<? extends OperatorDesc> op) {
+ FakeVectorDataSourceOperator parent = new FakeVectorDataSourceOperator(source);
+ List<Operator<? extends OperatorDesc>> listParents =
+ new ArrayList<Operator<? extends OperatorDesc>>(1);
+ listParents.add(parent);
+ op.setParentOperators(listParents);
+ List<Operator<? extends OperatorDesc>> listChildren =
+ new ArrayList<Operator<? extends OperatorDesc>>(1);
+ listChildren.add(op);
+ parent.setChildOperators(listChildren);
+ return parent;
+ }
+
+ public FakeVectorDataSourceOperator(
+ Iterable<VectorizedRowBatch> source) {
+ this.source = source;
+ }
+
+ @Override
+ public void initializeOp(Configuration conf) throws HiveException {
+ }
+
+ @Override
+ public void processOp(Object row, int tag) throws HiveException {
+ for (VectorizedRowBatch unit: source) {
+ forward(unit, null);
+ }
+ }
+
+ @Override
+ public OperatorType getType() {
+ return null;
+ }
+}