You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/10/29 20:40:27 UTC
[48/51] [partial] hive git commit: HIVE-17433: Vectorization: Support
Decimal64 in Hive Query Engine (Matt McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
index dea4db2..2966e56 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
@@ -46,20 +46,24 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- private int colNum;
- private <HiveOperandType1> value;
- private int outputColumn;
- private Timestamp scratchTimestamp2;
- private DateTimeMath dtm = new DateTimeMath();
+ private final <HiveOperandType1> value;
+ private final int colNum;
- public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumn) {
+ private transient final Timestamp scratchTimestamp2 = new Timestamp(0);
+ private transient final DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumnNum) {
+ super(outputColumnNum);
this.colNum = colNum;
this.value = value;
- this.outputColumn = outputColumn;
- scratchTimestamp2 = new Timestamp(0);
}
public <ClassName>() {
+ super();
+
+ // Dummy final assignments.
+ value = null;
+ colNum = -1;
}
@Override
@@ -78,7 +82,7 @@ public class <ClassName> extends VectorExpression {
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
// Output is type <ReturnType>.
- <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumnNum];
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
@@ -143,18 +147,8 @@ public class <ClassName> extends VectorExpression {
}
@Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "<ReturnType>";
- }
-
- @Override
public String vectorExpressionParameters() {
- return "val " + value.toString() + ", col " + + colNum;
+ return "val " + value.toString() + ", " + getColumnParamString(1, colNum);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
index e82b9e2..b6a5621 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
@@ -45,20 +45,24 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- private int colNum;
- private Timestamp value;
- private int outputColumn;
- private HiveIntervalYearMonth scratchIntervalYearMonth2;
- private DateTimeMath dtm = new DateTimeMath();
+ private final Timestamp value;
+ private final int colNum;
- public <ClassName>(Timestamp value, int colNum, int outputColumn) {
- this.colNum = colNum;
+ private transient final HiveIntervalYearMonth scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
+ private transient final DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(Timestamp value, int colNum, int outputColumnNum) {
+ super(outputColumnNum);
this.value = value;
- this.outputColumn = outputColumn;
- scratchIntervalYearMonth2 = new HiveIntervalYearMonth();
+ this.colNum = colNum;
}
public <ClassName>() {
+ super();
+
+ // Dummy final assignments.
+ value = null;
+ colNum = -1;
}
@Override
@@ -77,7 +81,7 @@ public class <ClassName> extends VectorExpression {
LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum];
// Output is type Timestamp.
- TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn];
+ TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
@@ -142,18 +146,8 @@ public class <ClassName> extends VectorExpression {
}
@Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-
- @Override
public String vectorExpressionParameters() {
- return "val " + value.toString() + ", col " + + colNum;
+ return "val " + value.toString() + ", " + getColumnParamString(1, colNum);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
index 0d8a26b..4050308 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
@@ -44,18 +44,23 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- private int colNum;
- private <HiveOperandType1> value;
- private int outputColumn;
- private DateTimeMath dtm = new DateTimeMath();
+ private final <HiveOperandType1> value;
+ private final int colNum;
- public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumn) {
- this.colNum = colNum;
+ private transient final DateTimeMath dtm = new DateTimeMath();
+
+ public <ClassName>(<HiveOperandType1> value, int colNum, int outputColumnNum) {
+ super(outputColumnNum);
this.value = value;
- this.outputColumn = outputColumn;
+ this.colNum = colNum;
}
public <ClassName>() {
+ super();
+
+ // Dummy final assignments.
+ value = null;
+ colNum = -1;
}
@Override
@@ -74,7 +79,7 @@ public class <ClassName> extends VectorExpression {
<InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum];
// Output is type <ReturnType>.
- <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumn];
+ <OutputColumnVectorType> outputColVector = (<OutputColumnVectorType>) batch.cols[outputColumnNum];
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
@@ -133,18 +138,8 @@ public class <ClassName> extends VectorExpression {
}
@Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "timestamp";
- }
-
- @Override
public String vectorExpressionParameters() {
- return "val " + value.toString() + ", col " + + colNum;
+ return "val " + value.toString() + ", " + getColumnParamString(1, colNum);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
index 6815b5b..d7e285e 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt
@@ -34,8 +34,8 @@ public class <ClassName> extends <BaseClassName> {
private static final long serialVersionUID = 1L;
- public <ClassName>(Timestamp value, int colNum, int outputColumn) {
- super(TimestampColumnVector.<GetTimestampLongDoubleMethod>(value), colNum, outputColumn);
+ public <ClassName>(Timestamp value, int colNum, int outputColumnNum) {
+ super(TimestampColumnVector.<GetTimestampLongDoubleMethod>(value), colNum, outputColumnNum);
}
public <ClassName>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
index ec0a395..33f7acd 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
@@ -37,17 +37,21 @@ public class <ClassName> extends VectorExpression {
private static final long serialVersionUID = 1L;
- private int colNum;
- private <HiveOperandType> value;
- private int outputColumn;
+ private final <HiveOperandType> value;
+ private final int colNum;
- public <ClassName>(<HiveOperandType> value, int colNum, int outputColumn) {
- this.colNum = colNum;
+ public <ClassName>(<HiveOperandType> value, int colNum, int outputColumnNum) {
+ super(outputColumnNum);
this.value = value;
- this.outputColumn = outputColumn;
+ this.colNum = colNum;
}
public <ClassName>() {
+ super();
+
+ // Dummy final assignments.
+ value = null;
+ colNum = -1;
}
@Override
@@ -60,7 +64,7 @@ public class <ClassName> extends VectorExpression {
// Input #2 is type <OperandType>.
<InputColumnVectorType> inputColVector2 = (<InputColumnVectorType>) batch.cols[colNum];
- LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn];
+ LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
boolean[] nullPos = inputColVector2.isNull;
@@ -125,18 +129,8 @@ public class <ClassName> extends VectorExpression {
}
@Override
- public int getOutputColumn() {
- return outputColumn;
- }
-
- @Override
- public String getOutputType() {
- return "long";
- }
-
- @Override
public String vectorExpressionParameters() {
- return "val " + value.toString() + ", col " + + colNum;
+ return "val " + value.toString() + ", " + getColumnParamString(1, colNum);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/ExpressionTemplates/TruncStringScalarCompareStringGroupColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TruncStringScalarCompareStringGroupColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TruncStringScalarCompareStringGroupColumn.txt
index 26da73a..89266c6 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TruncStringScalarCompareStringGroupColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TruncStringScalarCompareStringGroupColumn.txt
@@ -39,13 +39,12 @@ public class <ClassName> extends <BaseClassName> {
private static final long serialVersionUID = 1L;
- public <ClassName>(<TruncStringHiveType> value, int colNum, int outputColumn) {
- this.colNum = colNum;
- this.value = value.<TruncStringHiveGetBytes>;
- this.outputColumn = outputColumn;
+ public <ClassName>(<TruncStringHiveType> value, int colNum, int outputColumnNum) {
+ super(colNum, value.<TruncStringHiveGetBytes>, outputColumnNum);
}
public <ClassName>() {
+ super();
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
index a463373..d82cc3b 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
@@ -25,19 +25,17 @@ import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import com.google.common.base.Preconditions;
@@ -87,63 +85,24 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
-#IF PARTIAL1
- transient private Object[] partialResult;
- transient private LongWritable resultCount;
- transient private DoubleWritable resultSum;
- transient private <CamelCaseValueType>Writable resultInput;
- transient private StructObjectInspector soi;
-#ENDIF PARTIAL1
-#IF COMPLETE
- transient private DoubleWritable fullResult;
- transient private ObjectInspector oi;
-#ENDIF COMPLETE
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression, GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
#IF PARTIAL1
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL1);
#ENDIF PARTIAL1
#IF COMPLETE
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.COMPLETE);
#ENDIF COMPLETE
- }
+ init();
+ }
private void init() {
-#IF PARTIAL1
- partialResult = new Object[3];
- resultCount = new LongWritable();
- resultSum = new DoubleWritable();
- resultInput = new <CamelCaseValueType>Writable();
- partialResult[0] = resultCount;
- partialResult[1] = resultSum;
- partialResult[2] = resultInput;
- initPartialResultInspector();
-#ENDIF PARTIAL1
-#IF COMPLETE
- fullResult = new DoubleWritable();
- initFullResultInspector();
-#ENDIF COMPLETE
- }
-
-#IF PARTIAL1
- private void initPartialResultInspector() {
- List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
- foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writable<CamelCaseValueType>ObjectInspector);
- List<String> fname = new ArrayList<String>();
- fname.add("count");
- fname.add("sum");
- fname.add("input");
- soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
}
-#ENDIF PARTIAL1
-#IF COMPLETE
- private void initFullResultInspector() {
- oi = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
- }
-#ENDIF COMPLETE
private Aggregation getCurrentAggregationBuffer(
VectorAggregationBufferRow[] aggregationBufferSets,
@@ -168,8 +127,10 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
- <InputColumnVectorType> inputVector = ( <InputColumnVectorType>)batch.
- cols[this.inputExpression.getOutputColumn()];
+ <InputColumnVectorType> inputVector =
+ (<InputColumnVectorType>) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
<ValueType>[] vector = inputVector.vector;
if (inputVector.noNulls) {
@@ -345,7 +306,8 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
<InputColumnVectorType> inputVector =
- (<InputColumnVectorType>)batch.cols[this.inputExpression.getOutputColumn()];
+ (<InputColumnVectorType>) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
int batchSize = batch.size;
@@ -470,47 +432,78 @@ public class <ClassName> extends VectorAggregateExpression {
myAgg.reset();
}
- @Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- } else {
- Preconditions.checkState(myagg.count > 0);
+ @Override
+ public long getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Average input is <UpperCaseColumnVectorType>.
#IF PARTIAL1
- resultCount.set (myagg.count);
- resultSum.set (myagg.sum);
- return partialResult;
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL1.
#ENDIF PARTIAL1
#IF COMPLETE
- fullResult.set (myagg.sum / myagg.count);
- return fullResult;
+ * Output is DOUBLE.
+ *
+ * Mode COMPLETE.
#ENDIF COMPLETE
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.<UpperCaseColumnVectorType> &&
#IF PARTIAL1
- return soi;
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL1;
#ENDIF PARTIAL1
#IF COMPLETE
- return oi;
+ outputColVectorType == ColumnVector.Type.DOUBLE &&
+ mode == Mode.COMPLETE;
#ENDIF COMPLETE
}
@Override
- public long getAggregationBufferFixedSize() {
- JavaDataModel model = JavaDataModel.get();
- return JavaDataModel.alignUp(
- model.object() +
- model.primitive2() * 2,
- model.memoryAlign());
- }
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
- @Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
+#IF PARTIAL1
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL1
+#IF COMPLETE
+ DoubleColumnVector outputColVector = (DoubleColumnVector) batch.cols[columnNum];
+#ENDIF COMPLETE
+
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ Preconditions.checkState(myagg.count > 0);
+ outputColVector.isNull[batchIndex] = false;
+
+#IF PARTIAL1
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ ((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.sum;
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
+
+#ENDIF PARTIAL1
+#IF COMPLETE
+ outputColVector.vector[batchIndex] = myagg.sum / myagg.count;
+#ENDIF COMPLETE
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
index fa7b7c7..8613270 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
@@ -27,23 +27,21 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalUtil;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage.GenericUDAFAverageEvaluatorDecimal;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import com.google.common.base.Preconditions;
@@ -97,19 +95,12 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
-#IF PARTIAL1
- transient private Object[] partialResult;
- transient private LongWritable resultCount;
- transient private HiveDecimalWritable resultSum;
- transient private HiveDecimalWritable resultInput;
- transient private StructObjectInspector soi;
-#ENDIF PARTIAL1
#IF COMPLETE
transient private HiveDecimalWritable tempDecWritable;
- transient private HiveDecimalWritable fullResult;
- transient private ObjectInspector oi;
#ENDIF COMPLETE
+ DecimalTypeInfo outputDecimalTypeInfo;
+
/**
* The scale of the SUM in the partial output
*/
@@ -120,72 +111,34 @@ public class <ClassName> extends VectorAggregateExpression {
*/
private int sumPrecision;
- /**
- * the scale of the input expression
- */
- private int inputScale;
-
- /**
- * the precision of the input expression
- */
- private int inputPrecision;
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression,
- GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
#IF PARTIAL1
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL1);
#ENDIF PARTIAL1
#IF COMPLETE
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.COMPLETE);
#ENDIF COMPLETE
- }
+ init();
+ }
private void init() {
#IF PARTIAL1
- partialResult = new Object[3];
- resultCount = new LongWritable();
- resultSum = new HiveDecimalWritable();
- resultInput = new HiveDecimalWritable(0L);
- partialResult[0] = resultCount;
- partialResult[1] = resultSum;
- partialResult[2] = resultInput;
+ StructTypeInfo structTypeInfo = (StructTypeInfo) outputTypeInfo;
+ outputDecimalTypeInfo = (DecimalTypeInfo) structTypeInfo.getAllStructFieldTypeInfos().get(AVERAGE_SUM_FIELD_INDEX);
#ENDIF PARTIAL1
#IF COMPLETE
- tempDecWritable = new HiveDecimalWritable();
- fullResult = new HiveDecimalWritable();
+ outputDecimalTypeInfo = (DecimalTypeInfo) outputTypeInfo;
#ENDIF COMPLETE
- }
-
-#IF PARTIAL1
- private void initPartialResultInspector() {
-#ENDIF PARTIAL1
+ sumScale = outputDecimalTypeInfo.scale();
+ sumPrecision = outputDecimalTypeInfo.precision();
#IF COMPLETE
- private void initFullResultInspector() {
-#ENDIF COMPLETE
- // the output type of the vectorized partial aggregate must match the
- // expected type for the row-mode aggregation
- // For decimal, the type is "same number of integer digits and 4 more decimal digits"
-
- DecimalTypeInfo decTypeInfo =
- GenericUDAFAverageEvaluatorDecimal.deriveResultDecimalTypeInfo(
- inputPrecision, inputScale, mode);
- this.sumScale = decTypeInfo.scale();
- this.sumPrecision = decTypeInfo.precision();
-
-#IF PARTIAL1
- List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
- foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo));
- foi.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo));
- List<String> fname = new ArrayList<String>();
- fname.add("count");
- fname.add("sum");
- fname.add("input");
- soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
-#ENDIF PARTIAL1
-#IF COMPLETE
- oi = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo);
+ tempDecWritable = new HiveDecimalWritable();
#ENDIF COMPLETE
}
@@ -212,8 +165,10 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
- DecimalColumnVector inputVector = (DecimalColumnVector) batch.
- cols[this.inputExpression.getOutputColumn()];
+ DecimalColumnVector inputVector =
+ (DecimalColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
HiveDecimalWritable[] vector = inputVector.vector;
if (inputVector.noNulls) {
@@ -390,7 +345,8 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
DecimalColumnVector inputVector =
- (DecimalColumnVector)batch.cols[this.inputExpression.getOutputColumn()];
+ (DecimalColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
int batchSize = batch.size;
@@ -502,64 +458,86 @@ public class <ClassName> extends VectorAggregateExpression {
myAgg.reset();
}
- @Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- // !isSet checks for overflow.
- if (myagg.isNull || !myagg.sum.isSet()) {
- return null;
- }
- else {
- Preconditions.checkState(myagg.count > 0);
+ @Override
+ public long getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Average input is DECIMAL.
#IF PARTIAL1
- resultCount.set (myagg.count);
- resultSum.set(myagg.sum);
- return partialResult;
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL1.
#ENDIF PARTIAL1
#IF COMPLETE
- tempDecWritable.setFromLong (myagg.count);
- fullResult.set(myagg.sum);
- fullResult.mutateDivide(tempDecWritable);
- fullResult.mutateEnforcePrecisionScale(sumPrecision, sumScale);
- return fullResult;
+ * Output is DECIMAL.
+ *
+ * Mode COMPLETE.
#ENDIF COMPLETE
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.DECIMAL &&
#IF PARTIAL1
- return soi;
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL1;
#ENDIF PARTIAL1
#IF COMPLETE
- return oi;
+ outputColVectorType == ColumnVector.Type.DECIMAL &&
+ mode == Mode.COMPLETE;
#ENDIF COMPLETE
}
@Override
- public long getAggregationBufferFixedSize() {
- JavaDataModel model = JavaDataModel.get();
- return JavaDataModel.alignUp(
- model.object() +
- model.primitive2() * 2,
- model.memoryAlign());
- }
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
- @Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
+#IF PARTIAL1
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL1
+#IF COMPLETE
+ DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[columnNum];
+#ENDIF COMPLETE
- ExprNodeDesc inputExpr = desc.getParameters().get(0);
- DecimalTypeInfo tiInput = (DecimalTypeInfo) inputExpr.getTypeInfo();
- this.inputScale = tiInput.scale();
- this.inputPrecision = tiInput.precision();
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull || !myagg.sum.isSet()) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ Preconditions.checkState(myagg.count > 0);
+ outputColVector.isNull[batchIndex] = false;
#IF PARTIAL1
- initPartialResultInspector();
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ ((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex].set(myagg.sum);
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
+
#ENDIF PARTIAL1
#IF COMPLETE
- initFullResultInspector();
+ tempDecWritable.setFromLong (myagg.count);
+ HiveDecimalWritable result = outputColVector.vector[batchIndex];
+ result.set(myagg.sum);
+ result.mutateDivide(tempDecWritable);
+ result.mutateEnforcePrecisionScale(sumPrecision, sumScale);
+ if (!result.isSet()) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ }
#ENDIF COMPLETE
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
new file mode 100644
index 0000000..3b187b3
--- /dev/null
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
@@ -0,0 +1,551 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Generated from template VectorUDAFAvg.txt.
+ */
+@Description(name = "avg",
+ value = "_FUNC_(expr) - Returns the average value of expr (vectorized, type: decimal64)")
+public class <ClassName> extends VectorAggregateExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private int inputScale;
+
+ private DecimalTypeInfo outputDecimalTypeInfo;
+
+ private transient final HiveDecimalWritable temp = new HiveDecimalWritable();
+
+
+ /** class for storing the current aggregate value. */
+ static class Aggregation implements AggregationBuffer {
+
+ private static final long serialVersionUID = 1L;
+
+ // The max for 18 - 1 digits.
+ private static final long nearDecimal64Max =
+ HiveDecimalWritable.getDecimal64AbsMax(HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS - 1);
+
+ private final int inputScale;
+ private final HiveDecimalWritable temp;
+
+ //----------------------------------------------------------------------------------------------
+
+ private long sum;
+ private final HiveDecimalWritable regularDecimalSum = new HiveDecimalWritable(0);
+
+ transient private long count;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ private boolean isNull = true;
+ private boolean usingRegularDecimal = false;
+
+ public Aggregation(int inputScale, HiveDecimalWritable temp) {
+ this.inputScale = inputScale;
+ this.temp = temp;
+ }
+
+ public void avgValue(long value) {
+ if (isNull) {
+ sum = value;
+ count = 1;
+ isNull = false;
+ } else {
+ if (Math.abs(sum) > nearDecimal64Max) {
+ if (!usingRegularDecimal) {
+ usingRegularDecimal = true;
+ regularDecimalSum.deserialize64(sum, inputScale);
+ } else {
+ temp.deserialize64(sum, inputScale);
+ regularDecimalSum.mutateAdd(temp);
+ }
+ sum = value;
+ } else {
+ sum += value;
+ }
+
+ count++;
+ }
+ }
+
+ @Override
+ public int getVariableSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reset () {
+ isNull = true;
+ usingRegularDecimal = false;
+ sum = 0;
+ regularDecimalSum.setFromLong(0);
+
+ count = 0L;
+ }
+ }
+
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
+
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
+#IF PARTIAL1
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL1);
+#ENDIF PARTIAL1
+#IF COMPLETE
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.COMPLETE);
+#ENDIF COMPLETE
+ init();
+ }
+
+ private void init() {
+ inputScale = ((DecimalTypeInfo) inputTypeInfo).getScale();
+#IF PARTIAL1
+ StructTypeInfo structTypeInfo = (StructTypeInfo) outputTypeInfo;
+ outputDecimalTypeInfo = (DecimalTypeInfo) structTypeInfo.getAllStructFieldTypeInfos().get(AVERAGE_SUM_FIELD_INDEX);
+#ENDIF PARTIAL1
+#IF COMPLETE
+ outputDecimalTypeInfo = (DecimalTypeInfo) outputTypeInfo;
+#ENDIF COMPLETE
+ }
+
+ private Aggregation getCurrentAggregationBuffer(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ int row) {
+ VectorAggregationBufferRow mySet = aggregationBufferSets[row];
+ Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(bufferIndex);
+ return myagg;
+ }
+
+ @Override
+ public void aggregateInputSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ VectorizedRowBatch batch) throws HiveException {
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ inputExpression.evaluate(batch);
+
+ Decimal64ColumnVector inputVector =
+ (Decimal64ColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
+ long[] vector = inputVector.vector;
+
+ if (inputVector.noNulls) {
+ if (inputVector.isRepeating) {
+ iterateNoNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector[0], batchSize);
+ } else {
+ if (batch.selectedInUse) {
+ iterateNoNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector, batch.selected, batchSize);
+ } else {
+ iterateNoNullsWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector, batchSize);
+ }
+ }
+ } else {
+ if (inputVector.isRepeating) {
+ if (batch.selectedInUse) {
+ iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector[0], batchSize, batch.selected, inputVector.isNull);
+ } else {
+ iterateHasNullsRepeatingWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector[0], batchSize, inputVector.isNull);
+ }
+ } else {
+ if (batch.selectedInUse) {
+ iterateHasNullsSelectionWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector, batchSize, batch.selected, inputVector.isNull);
+ } else {
+ iterateHasNullsWithAggregationSelection(
+ aggregationBufferSets, bufferIndex,
+ vector, batchSize, inputVector.isNull);
+ }
+ }
+ }
+ }
+
+ private void iterateNoNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long value,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(value);
+ }
+ }
+
+ private void iterateNoNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long[] values,
+ int[] selection,
+ int batchSize) {
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(values[selection[i]]);
+ }
+ }
+
+ private void iterateNoNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long[] values,
+ int batchSize) {
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(values[i]);
+ }
+ }
+
+ private void iterateHasNullsRepeatingSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long value,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ if (isNull[0]) {
+ return;
+ }
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(value);
+ }
+
+ }
+
+ private void iterateHasNullsRepeatingWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long value,
+ int batchSize,
+ boolean[] isNull) {
+
+ if (isNull[0]) {
+ return;
+ }
+
+ for (int i=0; i < batchSize; ++i) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(value);
+ }
+ }
+
+ private void iterateHasNullsSelectionWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long[] values,
+ int batchSize,
+ int[] selection,
+ boolean[] isNull) {
+
+ for (int j=0; j < batchSize; ++j) {
+ int i = selection[j];
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ j);
+ myagg.avgValue(values[i]);
+ }
+ }
+ }
+
+ private void iterateHasNullsWithAggregationSelection(
+ VectorAggregationBufferRow[] aggregationBufferSets,
+ int bufferIndex,
+ long[] values,
+ int batchSize,
+ boolean[] isNull) {
+
+ for (int i=0; i < batchSize; ++i) {
+ if (!isNull[i]) {
+ Aggregation myagg = getCurrentAggregationBuffer(
+ aggregationBufferSets,
+ bufferIndex,
+ i);
+ myagg.avgValue(values[i]);
+ }
+ }
+ }
+
+ @Override
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
+ throws HiveException {
+
+ inputExpression.evaluate(batch);
+
+ Decimal64ColumnVector inputVector =
+ (Decimal64ColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
+ int batchSize = batch.size;
+
+ if (batchSize == 0) {
+ return;
+ }
+
+ Aggregation myagg = (Aggregation)agg;
+
+ long[] vector = inputVector.vector;
+
+ if (inputVector.isRepeating) {
+ if (inputVector.noNulls) {
+ final long value = vector[0];
+ for (int i = 0; i < batchSize; i++) {
+ myagg.avgValue(value);
+ }
+ }
+ return;
+ }
+
+ if (!batch.selectedInUse && inputVector.noNulls) {
+ iterateNoSelectionNoNulls(myagg, vector, batchSize);
+ } else if (!batch.selectedInUse) {
+ iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+ } else if (inputVector.noNulls){
+ iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+ } else {
+ iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected);
+ }
+ }
+
+ private void iterateSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull,
+ int[] selected) {
+
+ for (int j=0; j< batchSize; ++j) {
+ int i = selected[j];
+ if (!isNull[i]) {
+ myagg.avgValue(vector[i]);
+ }
+ }
+ }
+
+ private void iterateSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ int[] selected) {
+
+ for (int i=0; i< batchSize; ++i) {
+ myagg.avgValue(vector[selected[i]]);
+ }
+ }
+
+ private void iterateNoSelectionHasNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize,
+ boolean[] isNull) {
+
+ for(int i=0;i<batchSize;++i) {
+ if (!isNull[i]) {
+ myagg.avgValue(vector[i]);
+ }
+ }
+ }
+
+ private void iterateNoSelectionNoNulls(
+ Aggregation myagg,
+ long[] vector,
+ int batchSize) {
+
+ for (int i=0;i<batchSize;++i) {
+ myagg.avgValue(vector[i]);
+ }
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ return new Aggregation(inputScale, temp);
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ Aggregation myAgg = (Aggregation) agg;
+ myAgg.reset();
+ }
+
+ @Override
+ public long getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Average input is DECIMAL_64.
+#IF PARTIAL1
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL1.
+#ENDIF PARTIAL1
+#IF COMPLETE
+ * Output is DECIMAL.
+ *
+ * Mode COMPLETE.
+#ENDIF COMPLETE
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.DECIMAL_64 &&
+#IF PARTIAL1
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL1;
+#ENDIF PARTIAL1
+#IF COMPLETE
+ outputColVectorType == ColumnVector.Type.DECIMAL &&
+ mode == Mode.COMPLETE;
+#ENDIF COMPLETE
+ }
+
+ @Override
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
+
+#IF PARTIAL1
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL1
+#IF COMPLETE
+ DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[columnNum];
+#ENDIF COMPLETE
+
+ Aggregation myagg = (Aggregation) agg;
+ final boolean isNull;
+ if (!myagg.isNull) {
+ if (!myagg.usingRegularDecimal) {
+ myagg.regularDecimalSum.deserialize64(myagg.sum, inputScale);
+ } else {
+ myagg.temp.deserialize64(myagg.sum, inputScale);
+ myagg.regularDecimalSum.mutateAdd(myagg.temp);
+ }
+
+ isNull = !myagg.regularDecimalSum.isSet();
+ } else {
+ isNull = true;
+ }
+ if (isNull) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ Preconditions.checkState(myagg.count > 0);
+ outputColVector.isNull[batchIndex] = false;
+
+#IF PARTIAL1
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ ((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).set(
+ batchIndex, myagg.regularDecimalSum);
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
+
+#ENDIF PARTIAL1
+#IF COMPLETE
+ temp.setFromLong (myagg.count);
+ HiveDecimalWritable result = outputColVector.vector[batchIndex];
+ result.set(myagg.regularDecimalSum);
+ result.mutateDivide(temp);
+ result.mutateEnforcePrecisionScale(
+ outputDecimalTypeInfo.getPrecision(), outputDecimalTypeInfo.getScale());
+ if (!result.isSet()) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ }
+#ENDIF COMPLETE
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
index 071efc9..ad957df 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalUtil;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
@@ -37,17 +38,10 @@ import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage.GenericUDAFAverageEvaluatorDecimal;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import com.google.common.base.Preconditions;
@@ -101,22 +95,11 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
-#IF PARTIAL2
- transient private Object[] partialResult;
- transient private LongWritable resultCount;
- transient private HiveDecimalWritable resultSum;
- transient private HiveDecimalWritable resultInput;
- transient private StructObjectInspector soi;
-#ENDIF PARTIAL2
#IF FINAL
transient private HiveDecimalWritable tempDecWritable;
- transient private HiveDecimalWritable fullResult;
- transient private ObjectInspector oi;
#ENDIF FINAL
- private transient int countOffset;
- private transient int sumOffset;
- private transient int inputOffset;
+ DecimalTypeInfo outputDecimalTypeInfo;
/**
* The scale of the SUM in the partial output
@@ -128,73 +111,28 @@ public class <ClassName> extends VectorAggregateExpression {
*/
private int sumPrecision;
- /**
- * the scale of the input expression
- */
- private int inputScale;
-
- /**
- * the precision of the input expression
- */
- private int inputPrecision;
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression,
- GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
#IF PARTIAL2
- Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL2);
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL2);
#ENDIF PARTIAL2
#IF FINAL
- Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.FINAL);
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.FINAL);
#ENDIF FINAL
- }
+ init();
+ }
private void init() {
-#IF PARTIAL2
- partialResult = new Object[3];
- resultCount = new LongWritable();
- resultSum = new HiveDecimalWritable();
- resultInput = new HiveDecimalWritable(0L);
- partialResult[0] = resultCount;
- partialResult[1] = resultSum;
- partialResult[2] = resultInput;
-#ENDIF PARTIAL2
+ outputDecimalTypeInfo = (DecimalTypeInfo) outputTypeInfo;
+ sumScale = outputDecimalTypeInfo.scale();
+ sumPrecision = outputDecimalTypeInfo.precision();
#IF FINAL
tempDecWritable = new HiveDecimalWritable();
- fullResult = new HiveDecimalWritable();
-#ENDIF FINAL
- }
-
-#IF PARTIAL2
- private void initPartialResultInspector() {
-#ENDIF PARTIAL2
-#IF FINAL
- private void initFullResultInspector() {
-#ENDIF FINAL
-
- // the output type of the vectorized partial aggregate must match the
- // expected type for the row-mode aggregation
- // For decimal, the type is "same number of integer digits and 4 more decimal digits"
-
- DecimalTypeInfo decTypeInfo =
- GenericUDAFAverageEvaluatorDecimal.deriveResultDecimalTypeInfo(
- inputPrecision, inputScale, mode);
- this.sumScale = decTypeInfo.scale();
- this.sumPrecision = decTypeInfo.precision();
-
-#IF PARTIAL2
- List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
- foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo));
- foi.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo));
- List<String> fname = new ArrayList<String>();
- fname.add("count");
- fname.add("sum");
- fname.add("input");
- soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
-#ENDIF PARTIAL2
-#IF FINAL
- oi = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(decTypeInfo);
#ENDIF FINAL
}
@@ -222,11 +160,13 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
StructColumnVector inputStructColVector =
- (StructColumnVector) batch.cols[this.inputExpression.getOutputColumn()];
+ (StructColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
ColumnVector[] fields = inputStructColVector.fields;
- long[] countVector = ((LongColumnVector) fields[countOffset]).vector;
- HiveDecimalWritable[] sumVector = ((DecimalColumnVector) fields[sumOffset]).vector;
+ long[] countVector = ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector;
+ HiveDecimalWritable[] sumVector = ((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector;
if (inputStructColVector.noNulls) {
if (inputStructColVector.isRepeating) {
@@ -409,11 +349,13 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
StructColumnVector inputStructColVector =
- (StructColumnVector) batch.cols[this.inputExpression.getOutputColumn()];
+ (StructColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
ColumnVector[] fields = inputStructColVector.fields;
- long[] countVector = ((LongColumnVector) fields[countOffset]).vector;
- HiveDecimalWritable[] sumVector = ((DecimalColumnVector) fields[sumOffset]).vector;
+ long[] countVector = ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector;
+ HiveDecimalWritable[] sumVector = ((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector;
int batchSize = batch.size;
@@ -525,41 +467,6 @@ public class <ClassName> extends VectorAggregateExpression {
myAgg.reset();
}
- @Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- // !isSet checks for overflow.
- if (myagg.isNull || !myagg.mergeSum.isSet()) {
- return null;
- }
- else {
- Preconditions.checkState(myagg.mergeCount > 0);
-#IF PARTIAL2
- resultCount.set (myagg.mergeCount);
- resultSum.set(myagg.mergeSum);
- return partialResult;
-#ENDIF PARTIAL2
-#IF FINAL
- tempDecWritable.setFromLong (myagg.mergeCount);
- fullResult.set(myagg.mergeSum);
- fullResult.mutateDivide(tempDecWritable);
- fullResult.mutateEnforcePrecisionScale(sumPrecision, sumScale);
- return fullResult;
-#ENDIF FINAL
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
-#IF PARTIAL2
- return soi;
-#ENDIF PARTIAL2
-#IF FINAL
- return oi;
-#ENDIF FINAL
- }
-
@Override
public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
@@ -570,27 +477,75 @@ public class <ClassName> extends VectorAggregateExpression {
}
@Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
- ExprNodeDesc inputExpr = desc.getParameters().get(0);
+ /*
+ * Variance input is STRUCT.
+#IF PARTIAL2
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL2.
+#ENDIF PARTIAL2
+#IF FINAL
+ * Output is DECIMAL.
+ *
+ * Mode FINAL.
+#ENDIF FINAL
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.STRUCT &&
+#IF PARTIAL2
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL2;
+#ENDIF PARTIAL2
+#IF FINAL
+ outputColVectorType == ColumnVector.Type.DECIMAL &&
+ mode == Mode.FINAL;
+#ENDIF FINAL
+ }
- StructTypeInfo partialStructTypeInfo = (StructTypeInfo) inputExpr.getTypeInfo();
+ @Override
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
- ArrayList<String> fieldNames = partialStructTypeInfo.getAllStructFieldNames();
- countOffset = fieldNames.indexOf("count");
- sumOffset = fieldNames.indexOf("sum");
- inputOffset = fieldNames.indexOf("input");
+#IF PARTIAL2
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL2
+#IF FINAL
+ DecimalColumnVector outputColVector = (DecimalColumnVector) batch.cols[columnNum];
+#ENDIF FINAL
- DecimalTypeInfo tiInput = (DecimalTypeInfo) partialStructTypeInfo.getAllStructFieldTypeInfos().get(sumOffset);
- this.inputScale = tiInput.scale();
- this.inputPrecision = tiInput.precision();
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull || myagg.mergeCount == 0) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ outputColVector.isNull[batchIndex] = false;
#IF PARTIAL2
- initPartialResultInspector();
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.mergeCount;
+ ((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex].set(myagg.mergeSum);
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
+
#ENDIF PARTIAL2
#IF FINAL
- initFullResultInspector();
+ tempDecWritable.setFromLong (myagg.mergeCount);
+ HiveDecimalWritable result = outputColVector.vector[batchIndex];
+ result.set(myagg.mergeSum);
+ result.mutateDivide(tempDecWritable);
+ result.mutateEnforcePrecisionScale(sumPrecision, sumScale);
+ if (!result.isSet()) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ }
#ENDIF FINAL
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
index 996d0dc..cfcec3b 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
@@ -34,14 +35,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import com.google.common.base.Preconditions;
@@ -91,67 +86,24 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
-#IF PARTIAL2
- transient private Object[] partialResult;
- transient private LongWritable resultCount;
- transient private DoubleWritable resultSum;
- transient private DoubleWritable resultInput;
- transient private StructObjectInspector soi;
-#ENDIF PARTIAL2
-#IF FINAL
- transient private DoubleWritable fullResult;
-
- transient private ObjectInspector oi;
-#ENDIF FINAL
-
- private transient int countOffset;
- private transient int sumOffset;
- private transient int inputOffset;
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression, GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
#IF PARTIAL2
- Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL2);
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL2);
#ENDIF PARTIAL2
#IF FINAL
- Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.FINAL);
+ Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.FINAL);
#ENDIF FINAL
- }
+ init();
+ }
private void init() {
-#IF PARTIAL2
- partialResult = new Object[3];
- resultCount = new LongWritable();
- resultSum = new DoubleWritable();
- partialResult[0] = resultCount;
- partialResult[1] = resultSum;
- partialResult[2] = resultInput;
- initPartialResultInspector();
-#ENDIF PARTIAL2
-#IF FINAL
- fullResult = new DoubleWritable();
- initFullResultInspector();
-#ENDIF FINAL
- }
-
-#IF PARTIAL2
- private void initPartialResultInspector() {
- List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
- foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
- List<String> fname = new ArrayList<String>();
- fname.add("count");
- fname.add("sum");
- fname.add("input");
- soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
- }
-#ENDIF PARTIAL2
-#IF FINAL
- private void initFullResultInspector() {
- oi = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
-#ENDIF FINAL
private Aggregation getCurrentAggregationBuffer(
VectorAggregationBufferRow[] aggregationBufferSets,
@@ -177,11 +129,12 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
StructColumnVector inputStructColVector =
- (StructColumnVector) batch.cols[this.inputExpression.getOutputColumn()];
+ (StructColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
ColumnVector[] fields = inputStructColVector.fields;
- long[] countVector = ((LongColumnVector) fields[countOffset]).vector;
- double[] sumVector = ((DoubleColumnVector) fields[sumOffset]).vector;
+ long[] countVector = ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector;
+ double[] sumVector = ((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector;
if (inputStructColVector.noNulls) {
if (inputStructColVector.isRepeating) {
@@ -364,11 +317,13 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
StructColumnVector inputStructColVector =
- (StructColumnVector) batch.cols[this.inputExpression.getOutputColumn()];
+ (StructColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
+
ColumnVector[] fields = inputStructColVector.fields;
- long[] countVector = ((LongColumnVector) fields[countOffset]).vector;
- double[] sumVector = ((DoubleColumnVector) fields[sumOffset]).vector;
+ long[] countVector = ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector;
+ double[] sumVector = ((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector;
int batchSize = batch.size;
@@ -492,56 +447,77 @@ public class <ClassName> extends VectorAggregateExpression {
myAgg.reset();
}
- @Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- }
- else {
- assert(0 < myagg.mergeCount);
+ @Override
+ public long getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Variance input is STRUCT.
#IF PARTIAL2
- resultCount.set (myagg.mergeCount);
- resultSum.set (myagg.mergeSum);
- return partialResult;
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL2.
#ENDIF PARTIAL2
#IF FINAL
- fullResult.set (myagg.mergeSum / myagg.mergeCount);
- return fullResult;
+ * Output is DOUBLE.
+ *
+ * Mode FINAL.
#ENDIF FINAL
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.STRUCT &&
#IF PARTIAL2
- return soi;
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL2;
#ENDIF PARTIAL2
#IF FINAL
- return oi;
+ outputColVectorType == ColumnVector.Type.DOUBLE &&
+ mode == Mode.FINAL;
#ENDIF FINAL
}
@Override
- public long getAggregationBufferFixedSize() {
- JavaDataModel model = JavaDataModel.get();
- return JavaDataModel.alignUp(
- model.object() +
- model.primitive2() * 2,
- model.memoryAlign());
- }
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
- @Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
+#IF PARTIAL2
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL2
+#IF FINAL
+ DoubleColumnVector outputColVector = (DoubleColumnVector) batch.cols[columnNum];
+#ENDIF FINAL
- ExprNodeDesc inputExpr = desc.getParameters().get(0);
- StructTypeInfo partialStructTypeInfo = (StructTypeInfo) inputExpr.getTypeInfo();
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull || myagg.mergeCount == 0) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ outputColVector.isNull[batchIndex] = false;
+
+#IF PARTIAL2
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.mergeCount;
+ ((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.mergeSum;
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
- ArrayList<String> fieldNames = partialStructTypeInfo.getAllStructFieldNames();
- countOffset = fieldNames.indexOf("count");
- sumOffset = fieldNames.indexOf("sum");
- inputOffset = fieldNames.indexOf("input");
+#ENDIF PARTIAL2
+#IF FINAL
+ outputColVector.vector[batchIndex] = myagg.mergeSum / myagg.mergeCount;
+#ENDIF FINAL
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
index b816a35..dcbd1b4 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
@@ -24,20 +24,20 @@ import java.util.List;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.ql.util.TimestampUtils;
import com.google.common.base.Preconditions;
@@ -88,63 +88,24 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
-#IF PARTIAL1
- transient private Object[] partialResult;
- transient private LongWritable resultCount;
- transient private DoubleWritable resultSum;
- transient private TimestampWritable resultInput;
- transient private StructObjectInspector soi;
-#ENDIF PARTIAL1
-#IF COMPLETE
- transient private DoubleWritable fullResult;
- transient private ObjectInspector oi;
-#ENDIF COMPLETE
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression,
- GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
#IF PARTIAL1
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.PARTIAL1);
#ENDIF PARTIAL1
#IF COMPLETE
Preconditions.checkState(this.mode == GenericUDAFEvaluator.Mode.COMPLETE);
#ENDIF COMPLETE
- }
+ init();
+ }
private void init() {
-#IF PARTIAL1
- partialResult = new Object[3];
- resultCount = new LongWritable();
- resultSum = new DoubleWritable();
- resultInput = new TimestampWritable();
- partialResult[0] = resultCount;
- partialResult[1] = resultSum;
- partialResult[2] = resultInput;
- initPartialResultInspector();
-#ENDIF PARTIAL1
-#IF COMPLETE
- fullResult = new DoubleWritable();
-#ENDIF COMPLETE
- }
-
-#IF PARTIAL1
- private void initPartialResultInspector() {
- List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
- foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
- foi.add(PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
- List<String> fname = new ArrayList<String>();
- fname.add("count");
- fname.add("sum");
- fname.add("input");
- soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
}
-#ENDIF PARTIAL1
-#IF COMPLETE
- private void initFullResultInspector() {
- oi = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
- }
-#ENDIF COMPLETE
private Aggregation getCurrentAggregationBuffer(
VectorAggregationBufferRow[] aggregationBufferSets,
@@ -169,8 +130,9 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
- TimestampColumnVector inputColVector = (TimestampColumnVector)batch.
- cols[this.inputExpression.getOutputColumn()];
+ TimestampColumnVector inputColVector =
+ (TimestampColumnVector)batch.cols[
+ this.inputExpression.getOutputColumnNum()];
if (inputColVector.noNulls) {
if (inputColVector.isRepeating) {
@@ -343,7 +305,8 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
TimestampColumnVector inputColVector =
- (TimestampColumnVector)batch.cols[this.inputExpression.getOutputColumn()];
+ (TimestampColumnVector) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
int batchSize = batch.size;
@@ -469,49 +432,79 @@ public class <ClassName> extends VectorAggregateExpression {
myAgg.reset();
}
- @Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- }
- else {
- assert(0 < myagg.count);
+ @Override
+ public long getAggregationBufferFixedSize() {
+ JavaDataModel model = JavaDataModel.get();
+ return JavaDataModel.alignUp(
+ model.object() +
+ model.primitive2() * 2,
+ model.memoryAlign());
+ }
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Average input is TIMESTAMP.
#IF PARTIAL1
- resultCount.set(myagg.count);
- resultSum.set(myagg.sum);
- return partialResult;
+ * Output is STRUCT.
+ *
+ * Mode PARTIAL1.
#ENDIF PARTIAL1
#IF COMPLETE
- fullResult.set(myagg.sum / myagg.count);
- return fullResult;
+ * Output is DOUBLE.
+ *
+ * Mode COMPLETE.
#ENDIF COMPLETE
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
+ */
+ return
+ name.equals("avg") &&
+ inputColVectorType == ColumnVector.Type.TIMESTAMP &&
#IF PARTIAL1
- return soi;
+ outputColVectorType == ColumnVector.Type.STRUCT &&
+ mode == Mode.PARTIAL1;
#ENDIF PARTIAL1
#IF COMPLETE
- return oi;
+ outputColVectorType == ColumnVector.Type.DOUBLE &&
+ mode == Mode.COMPLETE;
#ENDIF COMPLETE
}
@Override
- public long getAggregationBufferFixedSize() {
- JavaDataModel model = JavaDataModel.get();
- return JavaDataModel.alignUp(
- model.object() +
- model.primitive2() * 2,
- model.memoryAlign());
- }
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
- @Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
+#IF PARTIAL1
+ StructColumnVector outputColVector = (StructColumnVector) batch.cols[columnNum];
+#ENDIF PARTIAL1
+#IF COMPLETE
+ DoubleColumnVector outputColVector = (DoubleColumnVector) batch.cols[columnNum];
+#ENDIF COMPLETE
+
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ Preconditions.checkState(myagg.count > 0);
+ outputColVector.isNull[batchIndex] = false;
+
+#IF PARTIAL1
+ ColumnVector[] fields = outputColVector.fields;
+ ((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ ((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.sum;
+
+ ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
+ sourceColVector.isRepeating = true;
+ sourceColVector.noNulls = false;
+ sourceColVector.isNull[0] = true;
+
+#ENDIF PARTIAL1
+#IF COMPLETE
+ outputColVector.vector[batchIndex] = myagg.sum / myagg.count;
+#ENDIF COMPLETE
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e63ebccc/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
index 81bd64f..f71f3a6 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
@@ -24,14 +24,16 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggreg
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* <ClassName>. Vectorized implementation for MIN/MAX aggregates.
@@ -84,23 +86,19 @@ public class <ClassName> extends VectorAggregateExpression {
}
}
- private transient VectorExpressionWriter resultWriter;
+ // This constructor is used to momentarily create the object so match can be called.
+ public <ClassName>() {
+ super();
+ }
- public <ClassName>(VectorExpression inputExpression, GenericUDAFEvaluator.Mode mode) {
- super(inputExpression, mode);
- }
+ public <ClassName>(VectorAggregationDesc vecAggrDesc) {
+ super(vecAggrDesc);
+ init();
+ }
private void init() {
}
- @Override
- public void init(AggregationDesc desc) throws HiveException {
- init();
-
- resultWriter = VectorExpressionWriterFactory.genVectorExpressionWritable(
- desc.getParameters().get(0));
- }
-
private Aggregation getCurrentAggregationBuffer(
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
@@ -124,8 +122,9 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
- <InputColumnVectorType> inputVector = (<InputColumnVectorType>)batch.
- cols[this.inputExpression.getOutputColumn()];
+ <InputColumnVectorType> inputVector =
+ (<InputColumnVectorType>) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
<ValueType>[] vector = inputVector.vector;
if (inputVector.noNulls) {
@@ -300,8 +299,9 @@ public class <ClassName> extends VectorAggregateExpression {
inputExpression.evaluate(batch);
- <InputColumnVectorType> inputVector = (<InputColumnVectorType>)batch.
- cols[this.inputExpression.getOutputColumn()];
+ <InputColumnVectorType> inputVector =
+ (<InputColumnVectorType>) batch.cols[
+ this.inputExpression.getOutputColumnNum()];
int batchSize = batch.size;
@@ -408,23 +408,6 @@ public class <ClassName> extends VectorAggregateExpression {
}
@Override
- public Object evaluateOutput(
- AggregationBuffer agg) throws HiveException {
- Aggregation myagg = (Aggregation) agg;
- if (myagg.isNull) {
- return null;
- }
- else {
- return resultWriter.writeValue(myagg.value);
- }
- }
-
- @Override
- public ObjectInspector getOutputObjectInspector() {
- return resultWriter.getObjectInspector();
- }
-
- @Override
public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
@@ -432,4 +415,34 @@ public class <ClassName> extends VectorAggregateExpression {
model.primitive2(),
model.memoryAlign());
}
+
+ @Override
+ public boolean matches(String name, ColumnVector.Type inputColVectorType,
+ ColumnVector.Type outputColVectorType, Mode mode) {
+
+ /*
+ * Min/max input and output is <UpperCaseColumnVectorType>.
+ *
+ * Any mode (PARTIAL1, PARTIAL2, FINAL, COMPLETE).
+ */
+ return
+ name.equals("<DescriptionName>") &&
+ inputColVectorType == ColumnVector.Type.<UpperCaseColumnVectorType> &&
+ outputColVectorType == ColumnVector.Type.<UpperCaseColumnVectorType>;
+ }
+
+ @Override
+ public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int columnNum,
+ AggregationBuffer agg) throws HiveException {
+
+ <InputColumnVectorType> outputColVector = (<InputColumnVectorType>) batch.cols[columnNum];
+ Aggregation myagg = (Aggregation) agg;
+ if (myagg.isNull) {
+ outputColVector.noNulls = false;
+ outputColVector.isNull[batchIndex] = true;
+ return;
+ }
+ outputColVector.isNull[batchIndex] = false;
+ outputColVector.vector[batchIndex] = myagg.value;
+ }
}