You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/05 17:31:53 UTC
[06/51] [partial] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index f68228c..82e8748 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -18,11 +18,15 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapper.EmptyVectorHashKeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
/**
* Class for handling vectorized hash map key wrappers. It evaluates the key columns in a
@@ -59,6 +63,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
*/
private int keysFixedSize;
+ /**
+ * Shared hashcontext for all keys in this batch
+ */
+ private final VectorHashKeyWrapper.HashContext hashCtx = new VectorHashKeyWrapper.HashContext();
+
/**
* Returns the compiled fixed size for the key wrappers.
* @return
@@ -85,12 +94,21 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* @throws HiveException
*/
public void evaluateBatch(VectorizedRowBatch batch) throws HiveException {
- for(int i = 0; i < keyExpressions.length; ++i) {
- keyExpressions[i].evaluate(batch);
+
+ if (keyCount == 0) {
+ // all keywrappers must be EmptyVectorHashKeyWrapper
+ return;
}
+
+ for(int i=0;i<batch.size;++i) {
+ vectorHashKeyWrappers[i].clearIsNull();
+ }
+
+ int keyIndex;
+ int columnIndex;
for(int i = 0; i< longIndices.length; ++i) {
- int keyIndex = longIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = longIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
LongColumnVector columnVector = (LongColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignLongNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -99,11 +117,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignLongNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignLongNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignLongNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignLongNullsRepeating(i, batch.size, columnVector);
+ assignLongNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignLongNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignLongNullsNoRepeatingSelection (keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented Long null/repeat/selected combination %b/%b/%b",
@@ -111,8 +129,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<doubleIndices.length; ++i) {
- int keyIndex = doubleIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = doubleIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
DoubleColumnVector columnVector = (DoubleColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDoubleNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -121,11 +139,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignDoubleNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignDoubleNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignDoubleNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignDoubleNullsRepeating(i, batch.size, columnVector);
+ assignDoubleNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignDoubleNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignDoubleNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented Double null/repeat/selected combination %b/%b/%b",
@@ -133,8 +151,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<stringIndices.length; ++i) {
- int keyIndex = stringIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = stringIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
BytesColumnVector columnVector = (BytesColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignStringNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -143,11 +161,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignStringNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignStringNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignStringNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignStringNullsRepeating(i, batch.size, columnVector);
+ assignStringNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignStringNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignStringNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented String null/repeat/selected combination %b/%b/%b",
@@ -155,8 +173,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<decimalIndices.length; ++i) {
- int keyIndex = decimalIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = decimalIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
DecimalColumnVector columnVector = (DecimalColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -165,11 +183,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignDecimalNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignDecimalNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignDecimalNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignDecimalNullsRepeating(i, batch.size, columnVector);
+ assignDecimalNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignDecimalNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignDecimalNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented Decimal null/repeat/selected combination %b/%b/%b",
@@ -177,8 +195,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<timestampIndices.length; ++i) {
- int keyIndex = timestampIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = timestampIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
TimestampColumnVector columnVector = (TimestampColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignTimestampNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -187,11 +205,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignTimestampNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignTimestampNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignTimestampNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignTimestampNullsRepeating(i, batch.size, columnVector);
+ assignTimestampNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignTimestampNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignTimestampNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented timestamp null/repeat/selected combination %b/%b/%b",
@@ -199,8 +217,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<intervalDayTimeIndices.length; ++i) {
- int keyIndex = intervalDayTimeIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ keyIndex = intervalDayTimeIndices[i];
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
IntervalDayTimeColumnVector columnVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
@@ -209,11 +227,198 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
} else if (columnVector.noNulls && columnVector.isRepeating) {
assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignIntervalDayTimeNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ assignIntervalDayTimeNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignIntervalDayTimeNullsRepeating(i, batch.size, columnVector);
+ assignIntervalDayTimeNullsRepeating(keyIndex, i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignIntervalDayTimeNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ assignIntervalDayTimeNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented intervalDayTime null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<batch.size;++i) {
+ vectorHashKeyWrappers[i].setHashKey();
+ }
+ }
+
+ public void evaluateBatchGroupingSets(VectorizedRowBatch batch,
+ boolean[] groupingSetsOverrideIsNulls) throws HiveException {
+
+ for(int i=0;i<batch.size;++i) {
+ vectorHashKeyWrappers[i].clearIsNull();
+ }
+ int keyIndex;
+ int columnIndex;
+ for(int i = 0; i< longIndices.length; ++i) {
+ keyIndex = longIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullLong(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ LongColumnVector columnVector = (LongColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignLongNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignLongNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignLongNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignLongNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignLongNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignLongNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented Long null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<doubleIndices.length; ++i) {
+ keyIndex = doubleIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullDouble(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ DoubleColumnVector columnVector = (DoubleColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDoubleNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDoubleNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignDoubleNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDoubleNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignDoubleNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDoubleNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented Double null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<stringIndices.length; ++i) {
+ keyIndex = stringIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullString(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ BytesColumnVector columnVector = (BytesColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignStringNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignStringNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignStringNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignStringNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignStringNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignStringNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented String null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<decimalIndices.length; ++i) {
+ keyIndex = decimalIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullDecimal(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ DecimalColumnVector columnVector = (DecimalColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented Decimal null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<timestampIndices.length; ++i) {
+ keyIndex = timestampIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullTimestamp(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ TimestampColumnVector columnVector = (TimestampColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignTimestampNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignTimestampNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignTimestampNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignTimestampNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignTimestampNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignTimestampNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented timestamp null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<intervalDayTimeIndices.length; ++i) {
+ keyIndex = intervalDayTimeIndices[i];
+ if (groupingSetsOverrideIsNulls[keyIndex]) {
+ final int batchSize = batch.size;
+ for(int r = 0; r < batchSize; ++r) {
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(keyIndex, i);
+ }
+ continue;
+ }
+ columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ IntervalDayTimeColumnVector columnVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNullsRepeating(keyIndex, i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingSelection(keyIndex, i, batch.size, columnVector, batch.selected);
} else {
throw new HiveException (String.format(
"Unimplemented intervalDayTime null/repeat/selected combination %b/%b/%b",
@@ -229,14 +434,15 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for string type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignStringNullsNoRepeatingSelection(int index, int size,
+ private void assignStringNullsNoRepeatingSelection(int keyIndex, int index, int size,
BytesColumnVector columnVector, int[] selected) {
for(int i=0; i<size; ++i) {
int row = selected[i];
if (columnVector.isNull[row]) {
- vectorHashKeyWrappers[i].assignNullString(index);
+ vectorHashKeyWrappers[i].assignNullString(keyIndex, index);
} else {
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[row],
columnVector.start[row],
columnVector.length[row]);
@@ -248,14 +454,15 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, possible nulls, repeat values.
*/
- private void assignStringNullsRepeating(int index, int size, BytesColumnVector columnVector) {
+ private void assignStringNullsRepeating(int keyIndex, int index, int size, BytesColumnVector columnVector) {
if (columnVector.isNull[0]) {
for(int i = 0; i < size; ++i) {
- vectorHashKeyWrappers[i].assignNullString(index);
+ vectorHashKeyWrappers[i].assignNullString(keyIndex, index);
}
} else {
for(int i = 0; i < size; ++i) {
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[0],
columnVector.start[0],
columnVector.length[0]);
@@ -267,13 +474,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for string type, possible nulls, no repeat values, no selection vector.
*/
- private void assignStringNullsNoRepeatingNoSelection(int index, int size,
+ private void assignStringNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
BytesColumnVector columnVector) {
for(int i=0; i<size; ++i) {
if (columnVector.isNull[i]) {
- vectorHashKeyWrappers[i].assignNullString(index);
+ vectorHashKeyWrappers[i].assignNullString(keyIndex, index);
} else {
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[i],
columnVector.start[i],
columnVector.length[i]);
@@ -285,9 +493,11 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, no nulls, repeat values, no selection vector.
*/
- private void assignStringNoNullsRepeating(int index, int size, BytesColumnVector columnVector) {
+ private void assignStringNoNullsRepeating(int index, int size,
+ BytesColumnVector columnVector) {
for(int i = 0; i < size; ++i) {
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[0],
columnVector.start[0],
columnVector.length[0]);
@@ -302,7 +512,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
BytesColumnVector columnVector, int[] selected) {
for(int i=0; i<size; ++i) {
int row = selected[i];
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[row],
columnVector.start[row],
columnVector.length[row]);
@@ -316,7 +527,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
private void assignStringNoNullsNoRepeatingNoSelection(int index, int size,
BytesColumnVector columnVector) {
for(int i=0; i<size; ++i) {
- vectorHashKeyWrappers[i].assignString(index,
+ vectorHashKeyWrappers[i].assignString(
+ index,
columnVector.vector[i],
columnVector.start[i],
columnVector.length[i]);
@@ -327,14 +539,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignDoubleNullsNoRepeatingSelection(int index, int size,
+ private void assignDoubleNullsNoRepeatingSelection(int keyIndex, int index, int size,
DoubleColumnVector columnVector, int[] selected) {
for(int i = 0; i < size; ++i) {
int row = selected[i];
if (!columnVector.isNull[row]) {
vectorHashKeyWrappers[i].assignDouble(index, columnVector.vector[row]);
} else {
- vectorHashKeyWrappers[i].assignNullDouble(index);
+ vectorHashKeyWrappers[i].assignNullDouble(keyIndex, index);
}
}
}
@@ -343,10 +555,10 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Double type, repeat null values.
*/
- private void assignDoubleNullsRepeating(int index, int size,
+ private void assignDoubleNullsRepeating(int keyIndex, int index, int size,
DoubleColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
- vectorHashKeyWrappers[r].assignNullDouble(index);
+ vectorHashKeyWrappers[r].assignNullDouble(keyIndex, index);
}
}
@@ -354,13 +566,13 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Double type, possible nulls, repeat values.
*/
- private void assignDoubleNullsNoRepeatingNoSelection(int index, int size,
+ private void assignDoubleNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
DoubleColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
if (!columnVector.isNull[r]) {
vectorHashKeyWrappers[r].assignDouble(index, columnVector.vector[r]);
} else {
- vectorHashKeyWrappers[r].assignNullDouble(index);
+ vectorHashKeyWrappers[r].assignNullDouble(keyIndex, index);
}
}
}
@@ -401,14 +613,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignLongNullsNoRepeatingSelection(int index, int size,
+ private void assignLongNullsNoRepeatingSelection(int keyIndex, int index, int size,
LongColumnVector columnVector, int[] selected) {
for(int i = 0; i < size; ++i) {
int row = selected[i];
if (!columnVector.isNull[row]) {
vectorHashKeyWrappers[i].assignLong(index, columnVector.vector[row]);
} else {
- vectorHashKeyWrappers[i].assignNullLong(index);
+ vectorHashKeyWrappers[i].assignNullLong(keyIndex, index);
}
}
}
@@ -417,10 +629,10 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, repeating nulls.
*/
- private void assignLongNullsRepeating(int index, int size,
+ private void assignLongNullsRepeating(int keyIndex, int index, int size,
LongColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
- vectorHashKeyWrappers[r].assignNullLong(index);
+ vectorHashKeyWrappers[r].assignNullLong(keyIndex, index);
}
}
@@ -428,13 +640,13 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for double type, possible nulls, no repeat values, no selection vector.
*/
- private void assignLongNullsNoRepeatingNoSelection(int index, int size,
+ private void assignLongNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
LongColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
if (!columnVector.isNull[r]) {
vectorHashKeyWrappers[r].assignLong(index, columnVector.vector[r]);
} else {
- vectorHashKeyWrappers[r].assignNullLong(index);
+ vectorHashKeyWrappers[r].assignNullLong(keyIndex, index);
}
}
}
@@ -475,14 +687,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Decimal type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignDecimalNullsNoRepeatingSelection(int index, int size,
+ private void assignDecimalNullsNoRepeatingSelection(int keyIndex, int index, int size,
DecimalColumnVector columnVector, int[] selected) {
for(int i = 0; i < size; ++i) {
int row = selected[i];
if (!columnVector.isNull[row]) {
vectorHashKeyWrappers[i].assignDecimal(index, columnVector.vector[row]);
} else {
- vectorHashKeyWrappers[i].assignNullDecimal(index);
+ vectorHashKeyWrappers[i].assignNullDecimal(keyIndex, index);
}
}
}
@@ -491,10 +703,10 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Decimal type, repeat null values.
*/
- private void assignDecimalNullsRepeating(int index, int size,
+ private void assignDecimalNullsRepeating(int keyIndex, int index, int size,
DecimalColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
- vectorHashKeyWrappers[r].assignNullDecimal(index);
+ vectorHashKeyWrappers[r].assignNullDecimal(keyIndex, index);
}
}
@@ -502,13 +714,13 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Decimal type, possible nulls, repeat values.
*/
- private void assignDecimalNullsNoRepeatingNoSelection(int index, int size,
+ private void assignDecimalNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
DecimalColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
if (!columnVector.isNull[r]) {
vectorHashKeyWrappers[r].assignDecimal(index, columnVector.vector[r]);
} else {
- vectorHashKeyWrappers[r].assignNullDecimal(index);
+ vectorHashKeyWrappers[r].assignNullDecimal(keyIndex, index);
}
}
}
@@ -549,14 +761,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Timestamp type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignTimestampNullsNoRepeatingSelection(int index, int size,
+ private void assignTimestampNullsNoRepeatingSelection(int keyIndex, int index, int size,
TimestampColumnVector columnVector, int[] selected) {
for(int i = 0; i < size; ++i) {
int row = selected[i];
if (!columnVector.isNull[row]) {
vectorHashKeyWrappers[i].assignTimestamp(index, columnVector, row);
} else {
- vectorHashKeyWrappers[i].assignNullTimestamp(index);
+ vectorHashKeyWrappers[i].assignNullTimestamp(keyIndex, index);
}
}
}
@@ -565,10 +777,10 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Timestamp type, repeat null values.
*/
- private void assignTimestampNullsRepeating(int index, int size,
+ private void assignTimestampNullsRepeating(int keyIndex, int index, int size,
TimestampColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
- vectorHashKeyWrappers[r].assignNullTimestamp(index);
+ vectorHashKeyWrappers[r].assignNullTimestamp(keyIndex, index);
}
}
@@ -576,13 +788,13 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for Timestamp type, possible nulls, repeat values.
*/
- private void assignTimestampNullsNoRepeatingNoSelection(int index, int size,
+ private void assignTimestampNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
TimestampColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
if (!columnVector.isNull[r]) {
vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, r);
} else {
- vectorHashKeyWrappers[r].assignNullTimestamp(index);
+ vectorHashKeyWrappers[r].assignNullTimestamp(keyIndex, index);
}
}
}
@@ -623,14 +835,14 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for IntervalDayTime type, possible nulls, no repeat values, batch selection vector.
*/
- private void assignIntervalDayTimeNullsNoRepeatingSelection(int index, int size,
+ private void assignIntervalDayTimeNullsNoRepeatingSelection(int keyIndex, int index, int size,
IntervalDayTimeColumnVector columnVector, int[] selected) {
for(int i = 0; i < size; ++i) {
int row = selected[i];
if (!columnVector.isNull[row]) {
vectorHashKeyWrappers[i].assignIntervalDayTime(index, columnVector, row);
} else {
- vectorHashKeyWrappers[i].assignNullIntervalDayTime(index);
+ vectorHashKeyWrappers[i].assignNullIntervalDayTime(keyIndex, index);
}
}
}
@@ -639,10 +851,10 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for IntervalDayTime type, repeat null values.
*/
- private void assignIntervalDayTimeNullsRepeating(int index, int size,
+ private void assignIntervalDayTimeNullsRepeating(int keyIndex, int index, int size,
IntervalDayTimeColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
- vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(keyIndex, index);
}
}
@@ -650,13 +862,13 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* Helper method to assign values from a vector column into the key wrapper.
* Optimized for IntervalDayTime type, possible nulls, repeat values.
*/
- private void assignIntervalDayTimeNullsNoRepeatingNoSelection(int index, int size,
+ private void assignIntervalDayTimeNullsNoRepeatingNoSelection(int keyIndex, int index, int size,
IntervalDayTimeColumnVector columnVector) {
for(int r = 0; r < size; ++r) {
if (!columnVector.isNull[r]) {
vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, r);
} else {
- vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(keyIndex, index);
}
}
}
@@ -693,13 +905,28 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
+ public static VectorHashKeyWrapperBatch compileKeyWrapperBatch(VectorExpression[] keyExpressions)
+ throws HiveException
+ {
+
+ final int size = keyExpressions.length;
+ ColumnVector.Type[] columnVectorTypes = new ColumnVector.Type[size];
+ for (int i = 0; i < size; i++) {
+ String typeName = VectorizationContext.mapTypeNameSynonyms(keyExpressions[i].getOutputType());
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ columnVectorTypes[i] = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
+ }
+ return compileKeyWrapperBatch(keyExpressions, columnVectorTypes);
+ }
+
/**
* Prepares a VectorHashKeyWrapperBatch to work for a specific set of keys.
* Computes the fast access lookup indices, preallocates all needed internal arrays.
* This step is done only once per query, not once per batch. The information computed now
* will be used to generate proper individual VectorKeyHashWrapper objects.
*/
- public static VectorHashKeyWrapperBatch compileKeyWrapperBatch(VectorExpression[] keyExpressions)
+ public static VectorHashKeyWrapperBatch compileKeyWrapperBatch(VectorExpression[] keyExpressions,
+ ColumnVector.Type[] columnVectorTypes)
throws HiveException {
VectorHashKeyWrapperBatch compiledKeyWrapperBatch = new VectorHashKeyWrapperBatch(keyExpressions.length);
compiledKeyWrapperBatch.keyExpressions = keyExpressions;
@@ -707,8 +934,8 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
compiledKeyWrapperBatch.keysFixedSize = 0;
// Inspect the output type of each key expression.
- for(int i=0; i < keyExpressions.length; ++i) {
- compiledKeyWrapperBatch.addKey(keyExpressions[i].getOutputType());
+ for(int i=0; i < columnVectorTypes.length; ++i) {
+ compiledKeyWrapperBatch.addKey(columnVectorTypes[i]);
}
compiledKeyWrapperBatch.finishAdding();
@@ -744,49 +971,54 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
public VectorHashKeyWrapper allocateKeyWrapper() {
- return VectorHashKeyWrapper.allocate(longIndices.length, doubleIndices.length,
- stringIndices.length, decimalIndices.length, timestampIndices.length,
- intervalDayTimeIndices.length);
+ return VectorHashKeyWrapper.allocate(hashCtx,
+ longIndices.length,
+ doubleIndices.length,
+ stringIndices.length,
+ decimalIndices.length,
+ timestampIndices.length,
+ intervalDayTimeIndices.length,
+ keyCount);
}
/**
* Get the row-mode writable object value of a key from a key wrapper
* @param keyOutputWriter
*/
- public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i,
+ public Object getWritableKeyValue(VectorHashKeyWrapper kw, int keyIndex,
VectorExpressionWriter keyOutputWriter)
throws HiveException {
- KeyLookupHelper klh = indexLookup[i];
- if (klh.longIndex >= 0) {
- return kw.getIsLongNull(klh.longIndex) ? null :
- keyOutputWriter.writeValue(kw.getLongValue(klh.longIndex));
- } else if (klh.doubleIndex >= 0) {
- return kw.getIsDoubleNull(klh.doubleIndex) ? null :
- keyOutputWriter.writeValue(kw.getDoubleValue(klh.doubleIndex));
- } else if (klh.stringIndex >= 0) {
- return kw.getIsBytesNull(klh.stringIndex) ? null :
- keyOutputWriter.writeValue(
- kw.getBytes(klh.stringIndex),
- kw.getByteStart(klh.stringIndex),
- kw.getByteLength(klh.stringIndex));
- } else if (klh.decimalIndex >= 0) {
- return kw.getIsDecimalNull(klh.decimalIndex)? null :
- keyOutputWriter.writeValue(
- kw.getDecimal(klh.decimalIndex));
- } else if (klh.timestampIndex >= 0) {
- return kw.getIsTimestampNull(klh.timestampIndex)? null :
- keyOutputWriter.writeValue(
- kw.getTimestamp(klh.timestampIndex));
- } else if (klh.intervalDayTimeIndex >= 0) {
- return kw.getIsIntervalDayTimeNull(klh.intervalDayTimeIndex)? null :
- keyOutputWriter.writeValue(
- kw.getIntervalDayTime(klh.intervalDayTimeIndex));
- } else {
- throw new HiveException(String.format(
- "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d %d",
- i, klh.longIndex, klh.doubleIndex, klh.stringIndex, klh.decimalIndex,
- klh.timestampIndex, klh.intervalDayTimeIndex));
+ if (kw.isNull(keyIndex)) {
+ return null;
+ }
+
+ ColumnVector.Type columnVectorType = columnVectorTypes[keyIndex];
+ int columnTypeSpecificIndex = columnTypeSpecificIndices[keyIndex];
+
+ switch (columnVectorType) {
+ case LONG:
+ return keyOutputWriter.writeValue(
+ kw.getLongValue(columnTypeSpecificIndex));
+ case DOUBLE:
+ return keyOutputWriter.writeValue(
+ kw.getDoubleValue(columnTypeSpecificIndex));
+ case BYTES:
+ return keyOutputWriter.writeValue(
+ kw.getBytes(columnTypeSpecificIndex),
+ kw.getByteStart(columnTypeSpecificIndex),
+ kw.getByteLength(columnTypeSpecificIndex));
+ case DECIMAL:
+ return keyOutputWriter.writeValue(
+ kw.getDecimal(columnTypeSpecificIndex));
+ case TIMESTAMP:
+ return keyOutputWriter.writeValue(
+ kw.getTimestamp(columnTypeSpecificIndex));
+ case INTERVAL_DAY_TIME:
+ return keyOutputWriter.writeValue(
+ kw.getIntervalDayTime(columnTypeSpecificIndex));
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
index 848fc8e..4e05fa3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
@@ -215,6 +215,9 @@ public class VectorMapJoinOperator extends VectorMapJoinBaseOperator {
}
}
+ for (VectorExpression ve : keyExpressions) {
+ ve.evaluate(inBatch);
+ }
keyWrapperBatch.evaluateBatch(inBatch);
keyValues = keyWrapperBatch.getVectorHashKeyWrappers();
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
index ac3363e..f8c4223 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
@@ -257,6 +257,9 @@ public class VectorSMBMapJoinOperator extends SMBMapJoinOperator implements Vect
}
}
+ for (VectorExpression ve : keyExpressions) {
+ ve.evaluate(inBatch);
+ }
keyWrapperBatch.evaluateBatch(inBatch);
keyValues = keyWrapperBatch.getVectorHashKeyWrappers();
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 5b0c2bf..c3940cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1359,7 +1359,7 @@ public class VectorizationContext {
return "arguments: " + Arrays.toString(args) + ", argument classes: " + argClasses.toString();
}
- private static int STACK_LENGTH_LIMIT = 15;
+ private static final int STACK_LENGTH_LIMIT = 15;
public static String getStackTraceAsSingleLine(Throwable e) {
StringBuilder sb = new StringBuilder();
@@ -1461,6 +1461,8 @@ public class VectorizationContext {
ve = getBetweenFilterExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFIn) {
ve = getInExpression(childExpr, mode, returnType);
+ } else if (udf instanceof GenericUDFWhen) {
+ ve = getWhenExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFOPPositive) {
ve = getIdentityExpression(childExpr);
} else if (udf instanceof GenericUDFCoalesce || udf instanceof GenericUDFNvl) {
@@ -2320,6 +2322,54 @@ public class VectorizationContext {
return createVectorExpression(cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
+ private boolean isColumnOrNonNullConst(ExprNodeDesc exprNodeDesc) {
+ if (exprNodeDesc instanceof ExprNodeColumnDesc) {
+ return true;
+ }
+ if (exprNodeDesc instanceof ExprNodeConstantDesc) {
+ String typeString = exprNodeDesc.getTypeString();
+ if (!typeString.equalsIgnoreCase("void")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private VectorExpression getWhenExpression(List<ExprNodeDesc> childExpr,
+ VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
+
+ if (mode != VectorExpressionDescriptor.Mode.PROJECTION) {
+ return null;
+ }
+ if (childExpr.size() != 3) {
+ // For now, we only optimize the 2 value case.
+ return null;
+ }
+
+ /*
+ * When we have 2 simple values:
+ * CASE WHEN boolExpr THEN column | const ELSE column | const END
+ * then we can convert to: IF (boolExpr THEN column | const ELSE column | const)
+ */
+ // CONSIDER: Adding a version of IfExpr* than can handle a non-column/const expression in the
+ // THEN or ELSE.
+ ExprNodeDesc exprNodeDesc1 = childExpr.get(1);
+ ExprNodeDesc exprNodeDesc2 = childExpr.get(2);
+ if (isColumnOrNonNullConst(exprNodeDesc1) &&
+ isColumnOrNonNullConst(exprNodeDesc2)) {
+ // Yes.
+ GenericUDFIf genericUDFIf = new GenericUDFIf();
+ return
+ getVectorExpressionForUdf(
+ genericUDFIf,
+ GenericUDFIf.class,
+ childExpr,
+ mode,
+ returnType);
+ }
+ return null; // Not handled by vector classes yet.
+ }
+
/*
* Return vector expression for a custom (i.e. not built-in) UDF.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java
new file mode 100644
index 0000000..5a8a825
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.lazy.LazyByte;
+import org.apache.hadoop.hive.serde2.lazy.LazyInteger;
+import org.apache.hadoop.hive.serde2.lazy.LazyLong;
+import org.apache.hadoop.hive.serde2.lazy.LazyShort;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+/**
+ * Cast a string to a long.
+ *
+ * If other functions besides cast need to take a string in and produce a long,
+ * you can subclass this class or convert it to a superclass, and
+ * implement different "func()" methods for each operation.
+ */
+public class CastStringToLong extends VectorExpression {
+ private static final long serialVersionUID = 1L;
+ int inputColumn;
+ int outputColumn;
+
+ private transient boolean integerPrimitiveCategoryKnown = false;
+ protected transient PrimitiveCategory integerPrimitiveCategory;
+
+ public CastStringToLong(int inputColumn, int outputColumn) {
+ super();
+ this.inputColumn = inputColumn;
+ this.outputColumn = outputColumn;
+ }
+
+ public CastStringToLong() {
+ super();
+ }
+
+ /**
+ * Convert input string to a long, at position i in the respective vectors.
+ */
+ protected void func(LongColumnVector outV, BytesColumnVector inV, int batchIndex) {
+
+ byte[] bytes = inV.vector[batchIndex];
+ final int start = inV.start[batchIndex];
+ final int length = inV.length[batchIndex];
+ try {
+
+ switch (integerPrimitiveCategory) {
+ case BOOLEAN:
+ {
+ boolean booleanValue;
+ int i = start;
+ if (length == 4) {
+ if ((bytes[i] == 'T' || bytes[i] == 't') &&
+ (bytes[i + 1] == 'R' || bytes[i + 1] == 'r') &&
+ (bytes[i + 2] == 'U' || bytes[i + 2] == 'u') &&
+ (bytes[i + 3] == 'E' || bytes[i + 3] == 'e')) {
+ booleanValue = true;
+ } else {
+ // No boolean value match for 4 char field.
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ } else if (length == 5) {
+ if ((bytes[i] == 'F' || bytes[i] == 'f') &&
+ (bytes[i + 1] == 'A' || bytes[i + 1] == 'a') &&
+ (bytes[i + 2] == 'L' || bytes[i + 2] == 'l') &&
+ (bytes[i + 3] == 'S' || bytes[i + 3] == 's') &&
+ (bytes[i + 4] == 'E' || bytes[i + 4] == 'e')) {
+ booleanValue = false;
+ } else {
+ // No boolean value match for 5 char field.
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ } else if (length == 1) {
+ byte b = bytes[start];
+ if (b == '1' || b == 't' || b == 'T') {
+ booleanValue = true;
+ } else if (b == '0' || b == 'f' || b == 'F') {
+ booleanValue = false;
+ } else {
+ // No boolean value match for extended 1 char field.
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ } else {
+ // No boolean value match for other lengths.
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ outV.vector[batchIndex] = (booleanValue ? 1 : 0);
+ }
+ break;
+ case BYTE:
+ if (!LazyUtils.isNumberMaybe(bytes, start, length)) {
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ outV.vector[batchIndex] = LazyByte.parseByte(bytes, start, length, 10);
+ break;
+ case SHORT:
+ if (!LazyUtils.isNumberMaybe(bytes, start, length)) {
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ outV.vector[batchIndex] = LazyShort.parseShort(bytes, start, length, 10);
+ break;
+ case INT:
+ if (!LazyUtils.isNumberMaybe(bytes, start, length)) {
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ outV.vector[batchIndex] = LazyInteger.parseInt(bytes, start, length, 10);
+ break;
+ case LONG:
+ if (!LazyUtils.isNumberMaybe(bytes, start, length)) {
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ return;
+ }
+ outV.vector[batchIndex] = LazyLong.parseLong(bytes, start, length, 10);
+ break;
+ default:
+ throw new Error("Unexpected primitive category " + integerPrimitiveCategory);
+ }
+ } catch (Exception e) {
+
+ // for any exception in conversion to integer, produce NULL
+ outV.noNulls = false;
+ outV.isNull[batchIndex] = true;
+ }
+ }
+
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (!integerPrimitiveCategoryKnown) {
+ String typeName = getOutputType().toLowerCase();
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ integerPrimitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
+ integerPrimitiveCategoryKnown = true;
+ }
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ BytesColumnVector inV = (BytesColumnVector) batch.cols[inputColumn];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn];
+
+ if (n == 0) {
+
+ // Nothing to do
+ return;
+ }
+
+ if (inV.noNulls) {
+ outV.noNulls = true;
+ if (inV.isRepeating) {
+ outV.isRepeating = true;
+ func(outV, inV, 0);
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ func(outV, inV, i);
+ }
+ outV.isRepeating = false;
+ } else {
+ for(int i = 0; i != n; i++) {
+ func(outV, inV, i);
+ }
+ outV.isRepeating = false;
+ }
+ } else {
+
+ // Handle case with nulls. Don't do function if the value is null,
+ // because the data may be undefined for a null value.
+ outV.noNulls = false;
+ if (inV.isRepeating) {
+ outV.isRepeating = true;
+ outV.isNull[0] = inV.isNull[0];
+ if (!inV.isNull[0]) {
+ func(outV, inV, 0);
+ }
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ outV.isNull[i] = inV.isNull[i];
+ if (!inV.isNull[i]) {
+ func(outV, inV, i);
+ }
+ }
+ outV.isRepeating = false;
+ } else {
+ System.arraycopy(inV.isNull, 0, outV.isNull, 0, n);
+ for(int i = 0; i != n; i++) {
+ if (!inV.isNull[i]) {
+ func(outV, inV, i);
+ }
+ }
+ outV.isRepeating = false;
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ public int getInputColumn() {
+ return inputColumn;
+ }
+
+ public void setInputColumn(int inputColumn) {
+ this.inputColumn = inputColumn;
+ }
+
+ @Override
+ public String vectorExpressionParameters() {
+ return "col " + inputColumn;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder();
+ b.setMode(VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(1)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY)
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN);
+ return b.build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
index 6383e8a..266365e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
@@ -39,8 +39,8 @@ public class CuckooSetBytes {
private int salt = 0;
private Random gen = new Random(676983475);
private int rehashCount = 0;
- private static long INT_MASK = 0x00000000ffffffffL;
- private static long BYTE_MASK = 0x00000000000000ffL;
+ private static final long INT_MASK = 0x00000000ffffffffL;
+ private static final long BYTE_MASK = 0x00000000000000ffL;
/**
* Allocate a new set to hold expectedSize values. Re-allocation to expand
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/OctetLength.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/OctetLength.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/OctetLength.java
new file mode 100644
index 0000000..3b41ed4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/OctetLength.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+public class OctetLength extends VectorExpression {
+ private static final long serialVersionUID = 1L;
+ private int colNum;
+ private int outputColumn;
+
+ public OctetLength(int colNum, int outputColumn) {
+ this();
+ this.colNum = colNum;
+ this.outputColumn = outputColumn;
+ }
+
+ public OctetLength() {
+ super();
+ }
+
+ // Calculate the length of the UTF-8 strings in input vector and place results in output vector.
+ @Override
+ public void evaluate(VectorizedRowBatch batch) {
+
+ if (childExpressions != null) {
+ super.evaluateChildren(batch);
+ }
+
+ BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum];
+ LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn];
+ int[] sel = batch.selected;
+ int n = batch.size;
+ int [] length = inputColVector.length;
+ long[] resultLen = outV.vector;
+
+ if (n == 0) {
+ //Nothing to do
+ return;
+ }
+
+ if (inputColVector.noNulls) {
+ outV.noNulls = true;
+ if (inputColVector.isRepeating) {
+ outV.isRepeating = true;
+ resultLen[0] = length[0];
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ resultLen[i] = length[i];
+ }
+ outV.isRepeating = false;
+ } else {
+ for(int i = 0; i != n; i++) {
+ resultLen[i] = length[i];
+ }
+ outV.isRepeating = false;
+ }
+ } else {
+
+ /*
+ * Handle case with nulls. Don't do function if the value is null, to save time,
+ * because calling the function can be expensive.
+ */
+ outV.noNulls = false;
+ if (inputColVector.isRepeating) {
+ outV.isRepeating = true;
+ outV.isNull[0] = inputColVector.isNull[0];
+ if (!inputColVector.isNull[0]) {
+ resultLen[0] = length[0];
+ }
+ } else if (batch.selectedInUse) {
+ for(int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!inputColVector.isNull[i]) {
+ resultLen[i] = length[i];
+ }
+ outV.isNull[i] = inputColVector.isNull[i];
+ }
+ outV.isRepeating = false;
+ } else {
+ for(int i = 0; i != n; i++) {
+ if (!inputColVector.isNull[i]) {
+ resultLen[i] = length[i];
+ }
+ outV.isNull[i] = inputColVector.isNull[i];
+ }
+ outV.isRepeating = false;
+ }
+ }
+ }
+
+ @Override
+ public int getOutputColumn() {
+ return outputColumn;
+ }
+
+ @Override
+ public String getOutputType() {
+ return "Long";
+ }
+
+ public int getColNum() {
+ return colNum;
+ }
+
+ public void setColNum(int colNum) {
+ this.colNum = colNum;
+ }
+
+ public void setOutputColumn(int outputColumn) {
+ this.outputColumn = outputColumn;
+ }
+
+ public String vectorExpressionParameters() {
+ return "col " + colNum;
+ }
+
+ @Override
+ public VectorExpressionDescriptor.Descriptor getDescriptor() {
+ VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder();
+ b.setMode(VectorExpressionDescriptor.Mode.PROJECTION)
+ .setNumArguments(1)
+ .setArgumentTypes(
+ VectorExpressionDescriptor.ArgumentType.STRING_FAMILY)
+ .setInputExpressionTypes(
+ VectorExpressionDescriptor.InputExpressionType.COLUMN);
+ return b.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
index 0866f63..7ab4473 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java
@@ -52,7 +52,7 @@ public abstract class VectorAggregateExpression implements Serializable {
public abstract Object evaluateOutput(AggregationBuffer agg) throws HiveException;
public abstract ObjectInspector getOutputObjectInspector();
- public abstract int getAggregationBufferFixedSize();
+ public abstract long getAggregationBufferFixedSize();
public boolean hasVariableSize() {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java
index 74e25ae..4aac9d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java
@@ -492,7 +492,7 @@ public class VectorUDAFAvgDecimal extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
index 483d9dc..365dcf6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java
@@ -464,7 +464,7 @@ public class VectorUDAFAvgTimestamp extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java
index 2139eae..52b05ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java
@@ -383,7 +383,7 @@ public class VectorUDAFBloomFilter extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
if (bitSetSize < 0) {
// Not pretty, but we need a way to get the size
try {
@@ -396,7 +396,7 @@ public class VectorUDAFBloomFilter extends VectorAggregateExpression {
// BloomFilter: object(BitSet: object(data: long[]), numBits: int, numHashFunctions: int)
JavaDataModel model = JavaDataModel.get();
- int bloomFilterSize = JavaDataModel.alignUp(model.object() + model.lengthForLongArrayOfSize(bitSetSize),
+ long bloomFilterSize = JavaDataModel.alignUp(model.object() + model.lengthForLongArrayOfSize(bitSetSize),
model.memoryAlign());
return JavaDataModel.alignUp(
model.object() + bloomFilterSize + model.primitive1() + model.primitive1(),
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java
index d2446d5..b986eb4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java
@@ -339,7 +339,7 @@ public class VectorUDAFBloomFilterMerge extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
if (aggBufferSize < 0) {
// Not pretty, but we need a way to get the size
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
index 494febc..cadb6dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java
@@ -259,7 +259,7 @@ public class VectorUDAFCount extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
index dec88cb..c489f8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java
@@ -385,7 +385,7 @@ public class VectorUDAFCountMerge extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
index 337ba0a..3b66030 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java
@@ -142,7 +142,7 @@ public class VectorUDAFCountStar extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
index 8cd3506..5388d37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java
@@ -508,7 +508,7 @@ public class VectorUDAFStdPopTimestamp extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
index 61d6977..1769dc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java
@@ -508,7 +508,7 @@ public class VectorUDAFStdSampTimestamp extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
index b10f66f..a37e3f6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
@@ -431,7 +431,7 @@ public class VectorUDAFSumDecimal extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object(),
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
index 2709b07..61cdeaa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java
@@ -508,7 +508,7 @@ public class VectorUDAFVarPopTimestamp extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
index 03dce1e..c375461 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java
@@ -508,7 +508,7 @@ public class VectorUDAFVarSampTimestamp extends VectorAggregateExpression {
}
@Override
- public int getAggregationBufferFixedSize() {
+ public long getAggregationBufferFixedSize() {
JavaDataModel model = JavaDataModel.get();
return JavaDataModel.alignUp(
model.object() +
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
index cb30413..c4d5113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
@@ -531,6 +531,8 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
protected void reloadHashTable(byte pos, int partitionId)
throws IOException, HiveException, SerDeException, ClassNotFoundException {
+ this.vectorMapJoinHashTable = null;
+
// The super method will reload a hash table partition of one of the small tables.
// Currently, for native vector map join it will only be one small table.
super.reloadHashTable(pos, partitionId);
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
index 6242daf..b5eab8b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
@@ -107,4 +107,9 @@ public abstract class VectorMapJoinFastBytesHashMap
// Share the same write buffers with our value store.
keyStore = new VectorMapJoinFastKeyStore(valueStore.writeBuffers());
}
+
+ @Override
+ public long getEstimatedMemorySize() {
+ return super.getEstimatedMemorySize() + valueStore.getEstimatedMemorySize() + keyStore.getEstimatedMemorySize();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
index 1a41961..e779762 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
@@ -97,4 +97,9 @@ public abstract class VectorMapJoinFastBytesHashMultiSet
keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize);
}
+
+ @Override
+ public long getEstimatedMemorySize() {
+ return super.getEstimatedMemorySize() + keyStore.getEstimatedMemorySize();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
index 331867c..d493319 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
@@ -84,4 +84,9 @@ public abstract class VectorMapJoinFastBytesHashSet
keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize);
}
+
+ @Override
+ public long getEstimatedMemorySize() {
+ return super.getEstimatedMemorySize() + keyStore.getEstimatedMemorySize();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
index b93e977..10bc902 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
import java.io.IOException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashTable;
@@ -218,4 +219,9 @@ public abstract class VectorMapJoinFastBytesHashTable
super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount);
allocateBucketArray();
}
+
+ @Override
+ public long getEstimatedMemorySize() {
+ return super.getEstimatedMemorySize() + JavaDataModel.get().lengthForLongArrayOfSize(slotTriples.length);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
index 9030e5f..1f182ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTable;
@@ -40,10 +41,10 @@ public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTab
protected int metricExpands;
// 2^30 (we cannot use Integer.MAX_VALUE which is 2^31-1).
- public static int HIGHEST_INT_POWER_OF_2 = 1073741824;
+ public static final int HIGHEST_INT_POWER_OF_2 = 1073741824;
- public static int ONE_QUARTER_LIMIT = HIGHEST_INT_POWER_OF_2 / 4;
- public static int ONE_SIXTH_LIMIT = HIGHEST_INT_POWER_OF_2 / 6;
+ public static final int ONE_QUARTER_LIMIT = HIGHEST_INT_POWER_OF_2 / 4;
+ public static final int ONE_SIXTH_LIMIT = HIGHEST_INT_POWER_OF_2 / 6;
public void throwExpandError(int limit, String dataTypeName) {
throw new RuntimeException(
@@ -88,4 +89,10 @@ public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTab
public int size() {
return keysAssigned;
}
+
+ @Override
+ public long getEstimatedMemorySize() {
+ JavaDataModel jdm = JavaDataModel.get();
+ return JavaDataModel.alignUp(10L * jdm.primitive1() + jdm.primitive2(), jdm.memoryAlign());
+ }
}