You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/02/16 15:52:42 UTC
[28/32] hive git commit: HIVE-18622: Vectorization: IF Statements,
Comparisons, and more do not handle NULLs correctly (Matt McCline,
reviewed by Sergey Shelukhin, Deepak Jaiswal, Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleColumn.txt
index 54a1a37..7e65b9b 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleColumn.txt
@@ -52,6 +52,12 @@ public class <ClassName> extends VectorExpression {
@Override
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -60,21 +66,12 @@ public class <ClassName> extends VectorExpression {
<InputColumnVectorType2> inputColVector2 = (<InputColumnVectorType2>) batch.cols[colNum2];
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
- int n = batch.size;
<OperandType>[] vector2 = inputColVector2.vector;
long[] outputVector = outputColVector.vector;
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
+ /*
+ * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately.
+ */
NullUtil.propagateNullsColCol(
inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
index 3bb95dd..b1e92e0 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt
@@ -18,7 +18,10 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -54,6 +57,12 @@ public class <ClassName> extends VectorExpression {
@Override
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -61,61 +70,85 @@ public class <ClassName> extends VectorExpression {
TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum];
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
- boolean[] nullPos = inputColVector.isNull;
- boolean[] outNulls = outputColVector.isNull;
- int n = batch.size;
+ boolean[] inputIsNull = inputColVector.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
long[] outputVector = outputColVector.vector;
- // return immediately if batch is empty
- if (n == 0) {
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
+
+ if (inputColVector.isRepeating) {
+ if (inputColVector.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ outputVector[0] = inputColVector.<GetTimestampLongDoubleMethod>(0) <OperatorSymbol> value ? 1 : 0;
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- outputColVector.isRepeating = false;
- outputColVector.noNulls = inputColVector.noNulls;
if (inputColVector.noNulls) {
- if (inputColVector.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- outputVector[0] = inputColVector.<GetTimestampLongDoubleMethod>(0) <OperatorSymbol> value ? 1 : 0;
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
- }
+ if (batch.selectedInUse) {
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ outputVector[i] =
+ inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputVector[i] =
+ inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
- outputVector[i] = inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
+ outputVector[i] =
+ inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
}
}
} else {
- if (inputColVector.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- outputVector[0] = inputColVector.<GetTimestampLongDoubleMethod>(0) <OperatorSymbol> value ? 1 : 0;
- outNulls[0] = false;
- } else {
- outNulls[0] = true;
- }
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
+ if (batch.selectedInUse) {
for(int j=0; j != n; j++) {
int i = sel[j];
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
- outNulls[i] = false;
} else {
- //comparison with null is null
- outNulls[i] = true;
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
} else {
- System.arraycopy(nullPos, 0, outNulls, 0, n);
for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector.<GetTimestampLongDoubleMethod>(i) <OperatorSymbol> value ? 1 : 0;
+ } else {
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
index 3db5d01..b81b805 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt
@@ -55,6 +55,12 @@ public class <ClassName> extends VectorExpression {
@Override
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -67,20 +73,11 @@ public class <ClassName> extends VectorExpression {
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
- int n = batch.size;
long[] outputVector = outputColVector.vector;
- // return immediately if batch is empty
- if (n == 0) {
- return;
- }
-
- outputColVector.isRepeating =
- inputColVector1.isRepeating && inputColVector2.isRepeating
- || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0]
- || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0];
-
- // Handle nulls first
+ /*
+ * Propagate null values for a two-input operator and set isRepeating and noNulls appropriately.
+ */
NullUtil.propagateNullsColCol(
inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
index 1ee7b11..cee680a 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt
@@ -18,10 +18,12 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.*;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
@@ -55,6 +57,12 @@ public class <ClassName> extends VectorExpression {
@Override
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -65,61 +73,82 @@ public class <ClassName> extends VectorExpression {
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
- boolean[] nullPos = inputColVector1.isNull;
- boolean[] outNulls = outputColVector.isNull;
- int n = batch.size;
+ boolean[] inputIsNull = inputColVector1.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
long[] outputVector = outputColVector.vector;
- // return immediately if batch is empty
- if (n == 0) {
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
+
+ if (inputColVector1.isRepeating) {
+ if (inputColVector1.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- outputColVector.isRepeating = false;
- outputColVector.noNulls = inputColVector1.noNulls;
if (inputColVector1.noNulls) {
- if (inputColVector1.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- }
+ if (batch.selectedInUse) {
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
}
}
- } else {
- if (inputColVector1.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- outputVector[0] = inputColVector1.compareTo(0, value) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[0] = false;
- } else {
- outNulls[0] = true;
- }
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
+ } else /* there are NULLs in the inputColVector */ {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
+ if (batch.selectedInUse) {
for(int j=0; j != n; j++) {
int i = sel[j];
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[i] = false;
} else {
- //comparison with null is null
- outNulls[i] = true;
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
} else {
- System.arraycopy(nullPos, 0, outNulls, 0, n);
for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector1.compareTo(i, value) <OperatorSymbol> 0 ? 1 : 0;
+ } else {
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
index 509f264..b50cbc8 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
@@ -74,6 +75,12 @@ public class <ClassName> extends VectorExpression {
*/
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -87,34 +94,59 @@ public class <ClassName> extends VectorExpression {
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
+
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
long[] vector2 = inputColVector2.vector;
- // return immediately if batch is empty
- if (n == 0) {
+ if (inputColVector2.isRepeating) {
+ if (inputColVector2.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- if (inputColVector2.isRepeating) {
- scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0]));
- dtm.<OperatorMethod>(
- value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(0);
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
+ if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
- dtm.<OperatorMethod>(
- value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
- }
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);;
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
dtm.<OperatorMethod>(
@@ -122,24 +154,39 @@ public class <ClassName> extends VectorExpression {
outputColVector.setFromScratch<CamelReturnType>(i);
}
}
- } else { /* there are nulls */
+ } else /* there are NULLs in the inputColVector */ {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
- dtm.<OperatorMethod>(
- value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
- outputIsNull[i] = inputIsNull[i];
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
} else {
for(int i = 0; i != n; i++) {
- scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
- dtm.<OperatorMethod>(
- value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i]));
+ dtm.<OperatorMethod>(
+ value, scratchTimestamp2, outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
index 2de3044..9db7b53 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
@@ -73,6 +74,12 @@ public class <ClassName> extends VectorExpression {
*/
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -86,59 +93,99 @@ public class <ClassName> extends VectorExpression {
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
+
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
long[] vector2 = inputColVector2.vector;
- // return immediately if batch is empty
- if (n == 0) {
+ if (inputColVector2.isRepeating) {
+ if (inputColVector2.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ scratchIntervalYearMonth2.set((int) vector2[0]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(0);
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- if (inputColVector2.isRepeating) {
- scratchIntervalYearMonth2.set((int) vector2[0]);
- dtm.<OperatorMethod>(
- value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
- outputColVector.setFromScratchTimestamp(0);
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
+ if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- scratchIntervalYearMonth2.set((int) vector2[i]);
- dtm.<OperatorMethod>(
- value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
- outputColVector.setFromScratchTimestamp(i);
- }
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
scratchIntervalYearMonth2.set((int) vector2[i]);
dtm.<OperatorMethod>(
- value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
outputColVector.setFromScratchTimestamp(i);
}
}
- } else { /* there are nulls */
+ } else /* there are NULLs in the inputColVector */ {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- scratchIntervalYearMonth2.set((int) vector2[i]);
- dtm.<OperatorMethod>(
- value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
- outputColVector.setFromScratchTimestamp(i);
- outputIsNull[i] = inputIsNull[i];
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
} else {
for(int i = 0; i != n; i++) {
- scratchIntervalYearMonth2.set((int) vector2[i]);
- dtm.<OperatorMethod>(
- value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
- outputColVector.setFromScratchTimestamp(i);
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ scratchIntervalYearMonth2.set((int) vector2[i]);
+ dtm.<OperatorMethod>(
+ value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp());
+ outputColVector.setFromScratchTimestamp(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
index 4ed80d1..e860e4d 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
@@ -71,6 +72,12 @@ public class <ClassName> extends VectorExpression {
*/
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -84,53 +91,91 @@ public class <ClassName> extends VectorExpression {
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector2.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector2.noNulls;
- outputColVector.isRepeating = inputColVector2.isRepeating;
- int n = batch.size;
- // return immediately if batch is empty
- if (n == 0) {
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
+
+ if (inputColVector2.isRepeating) {
+ if (inputColVector2.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(0);
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- if (inputColVector2.isRepeating) {
- dtm.<OperatorMethod>(
- value, inputColVector2.asScratch<CamelOperandType2>(0), outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(0);
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
- } else if (inputColVector2.noNulls) {
+ if (inputColVector2.noNulls) {
if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- dtm.<OperatorMethod>(
- value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
- }
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
dtm.<OperatorMethod>(
- value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
outputColVector.setFromScratch<CamelReturnType>(i);
-
}
}
- } else { /* there are nulls */
+ } else /* there are NULLs in the inputColVector */ {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- dtm.<OperatorMethod>(
- value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
- outputIsNull[i] = inputIsNull[i];
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
} else {
for(int i = 0; i != n; i++) {
- dtm.<OperatorMethod>(
- value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
- outputColVector.setFromScratch<CamelReturnType>(i);
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
+ dtm.<OperatorMethod>(
+ value, inputColVector2.asScratch<CamelOperandType2>(i), outputColVector.getScratch<CamelReturnType>());
+ outputColVector.setFromScratch<CamelReturnType>(i);
+ } else {
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
+ }
}
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
index 6cca0bb..10f6162 100644
--- a/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
+++ b/ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt
@@ -18,10 +18,12 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+import java.util.Arrays;
import java.sql.Timestamp;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.*;
@@ -57,6 +59,12 @@ public class <ClassName> extends VectorExpression {
@Override
public void evaluate(VectorizedRowBatch batch) {
+ // return immediately if batch is empty
+ final int n = batch.size;
+ if (n == 0) {
+ return;
+ }
+
if (childExpressions != null) {
super.evaluateChildren(batch);
}
@@ -67,61 +75,84 @@ public class <ClassName> extends VectorExpression {
LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
int[] sel = batch.selected;
- boolean[] nullPos = inputColVector2.isNull;
- boolean[] outNulls = outputColVector.isNull;
- int n = batch.size;
+ boolean[] inputIsNull = inputColVector2.isNull;
+ boolean[] outputIsNull = outputColVector.isNull;
+
long[] outputVector = outputColVector.vector;
- // return immediately if batch is empty
- if (n == 0) {
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
+
+ if (inputColVector2.isRepeating) {
+ if (inputColVector2.noNulls || !inputIsNull[0]) {
+ outputIsNull[0] = false;
+ outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
+ outputColVector.isRepeating = true;
+ NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n);
return;
}
- outputColVector.isRepeating = false;
- outputColVector.noNulls = inputColVector2.noNulls;
if (inputColVector2.noNulls) {
- if (inputColVector2.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for(int j=0; j != n; j++) {
- int i = sel[j];
- outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- }
+ if (batch.selectedInUse) {
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ }
+ }
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
}
}
- } else {
- if (inputColVector2.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- if (!nullPos[0]) {
- outputVector[0] = inputColVector2.compareTo(value, 0) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[0] = false;
- } else {
- outNulls[0] = true;
- }
- outputColVector.isRepeating = true;
- } else if (batch.selectedInUse) {
+ } else /* there are NULLs in the inputColVector */ {
+
+ /*
+ * Do careful maintenance of the outputColVector.noNulls flag.
+ */
+
+ if (batch.selectedInUse) {
for(int j=0; j != n; j++) {
int i = sel[j];
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
- outNulls[i] = false;
} else {
- //comparison with null is null
- outNulls[i] = true;
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
} else {
- System.arraycopy(nullPos, 0, outNulls, 0, n);
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
for(int i = 0; i != n; i++) {
- if (!nullPos[i]) {
+ if (!inputIsNull[i]) {
+ outputIsNull[i] = false;
outputVector[i] = inputColVector2.compareTo(value, i) <OperatorSymbol> 0 ? 1 : 0;
+ } else {
+ // Comparison with NULL is NULL.
+ outputIsNull[i] = true;
+ outputColVector.noNulls = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/TestTemplates/TestColumnColumnOperationVectorExpressionEvaluation.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/TestTemplates/TestColumnColumnOperationVectorExpressionEvaluation.txt b/ql/src/gen/vectorization/TestTemplates/TestColumnColumnOperationVectorExpressionEvaluation.txt
index 4ab3e76..3c8f8822 100644
--- a/ql/src/gen/vectorization/TestTemplates/TestColumnColumnOperationVectorExpressionEvaluation.txt
+++ b/ql/src/gen/vectorization/TestTemplates/TestColumnColumnOperationVectorExpressionEvaluation.txt
@@ -48,9 +48,13 @@
|| inputColumnVector1.isRepeating && inputColumnVector2.isRepeating,
outputColumnVector.isRepeating);
+ /*
+ We no longer set noNulls to the input ColumnVector's value since that doesn't work
+ for scratch column reuse.
assertEquals(
"Output column vector no nulls state does not match operand columns",
inputColumnVector1.noNulls && inputColumnVector2.noNulls, outputColumnVector.noNulls);
+ */
//if repeating, only the first value matters
if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionCheckedEvaluation.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionCheckedEvaluation.txt b/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionCheckedEvaluation.txt
index df4f89d..c918f3b 100644
--- a/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionCheckedEvaluation.txt
+++ b/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionCheckedEvaluation.txt
@@ -46,9 +46,13 @@
"Output column vector is repeating state does not match operand column",
inputColumnVector.isRepeating, outputColumnVector.isRepeating);
+ /*
+ We no longer set noNulls to the input ColumnVector's value since that doesn't work
+ for scratch column reuse.
assertEquals(
"Output column vector no nulls state does not match operand column",
inputColumnVector.noNulls, outputColumnVector.noNulls);
+ */
if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
for(int i = 0; i < BATCH_SIZE; i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionEvaluation.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionEvaluation.txt b/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionEvaluation.txt
index e5f3f18..991135c 100644
--- a/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionEvaluation.txt
+++ b/ql/src/gen/vectorization/TestTemplates/TestColumnScalarOperationVectorExpressionEvaluation.txt
@@ -45,9 +45,13 @@
"Output column vector is repeating state does not match operand column",
inputColumnVector.isRepeating, outputColumnVector.isRepeating);
+ /*
+ We no longer set noNulls to the input ColumnVector's value since that doesn't work
+ for scratch column reuse.
assertEquals(
"Output column vector no nulls state does not match operand column",
inputColumnVector.noNulls, outputColumnVector.noNulls);
+ */
if(!outputColumnVector.noNulls && !outputColumnVector.isRepeating) {
for(int i = 0; i < BATCH_SIZE; i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
index 733731f..fc3d01f 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
@@ -320,7 +320,7 @@ public class <ClassName> extends VectorAggregateExpression {
<ValueType>[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.sum = 0;
@@ -493,9 +493,12 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL1
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.sum;
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
index 6e42598..fa72171 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal.txt
@@ -359,7 +359,7 @@ public class <ClassName> extends VectorAggregateExpression {
HiveDecimalWritable[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.sum.setFromLong(0L);
@@ -519,9 +519,12 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL1
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex].set(myagg.sum);
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
index d5325c3..53dceeb 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimal64ToDecimal.txt
@@ -371,7 +371,7 @@ public class <ClassName> extends VectorAggregateExpression {
long[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
final long value = vector[0];
for (int i = 0; i < batchSize; i++) {
myagg.avgValue(value);
@@ -525,10 +525,13 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL1
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).set(
batchIndex, myagg.regularDecimalSum);
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
index 8ab393c..e273d07 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgDecimalMerge.txt
@@ -368,7 +368,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputStructColVector.isRepeating) {
- if (inputStructColVector.noNulls) {
+ if (inputStructColVector.noNulls || !inputStructColVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.mergeSum.setFromLong(0L);
@@ -529,9 +529,12 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL2
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.mergeCount;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DecimalColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex].set(myagg.mergeSum);
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
index be2fadd..162d1ba 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgMerge.txt
@@ -334,7 +334,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputStructColVector.isRepeating) {
- if (inputStructColVector.noNulls) {
+ if (inputStructColVector.noNulls || !inputStructColVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.mergeCount = 0;
@@ -507,9 +507,12 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL2
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.mergeCount;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.mergeSum;
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
index 6190a9e..abb7b22 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvgTimestamp.txt
@@ -317,7 +317,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputColVector.isRepeating) {
- if (inputColVector.noNulls) {
+ if (inputColVector.noNulls || !inputColVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.sum = 0;
@@ -493,9 +493,12 @@ public class <ClassName> extends VectorAggregateExpression {
#IF PARTIAL1
ColumnVector[] fields = outputColVector.fields;
+ fields[AVERAGE_COUNT_FIELD_INDEX].isNull[batchIndex] = false;
((LongColumnVector) fields[AVERAGE_COUNT_FIELD_INDEX]).vector[batchIndex] = myagg.count;
+ fields[AVERAGE_SUM_FIELD_INDEX].isNull[batchIndex] = false;
((DoubleColumnVector) fields[AVERAGE_SUM_FIELD_INDEX]).vector[batchIndex] = myagg.sum;
+ // NULL out useless source field.
ColumnVector sourceColVector = (ColumnVector) fields[AVERAGE_SOURCE_FIELD_INDEX];
sourceColVector.isRepeating = true;
sourceColVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
index fd54256..2df45bb 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
@@ -314,7 +314,7 @@ public class <ClassName> extends VectorAggregateExpression {
<ValueType>[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
myagg.minmaxValue(vector[0]);
}
return;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
index 4764a45..9fe85d3 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
@@ -318,7 +318,7 @@ public class <ClassName> extends VectorAggregateExpression {
HiveDecimalWritable[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls &&
+ if ((inputVector.noNulls || !inputVector.isNull[0]) &&
(myagg.isNull || (myagg.value.compareTo(vector[0]) <OperatorSymbol> 0))) {
myagg.isNull = false;
myagg.value.set(vector[0]);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
index 4680161..9a0a6e7 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt
@@ -307,7 +307,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputColVector.isRepeating) {
- if (inputColVector.noNulls &&
+ if ((inputColVector.noNulls || !inputColVector.isNull[0]) &&
(myagg.isNull || (inputColVector.compareTo(myagg.value, 0) <OperatorSymbol> 0))) {
myagg.isNull = false;
inputColVector.intervalDayTimeUpdate(myagg.value, 0);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt
index 027688d..4f0b5a5 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt
@@ -278,7 +278,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputColumn.isRepeating) {
- if (inputColumn.noNulls) {
+ if (inputColumn.noNulls || !inputColumn.isNull[0]) {
myagg.checkValue(inputColumn.vector[0],
inputColumn.start[0],
inputColumn.length[0]);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
index 370b6a8..579437e 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt
@@ -309,7 +309,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputColVector.isRepeating) {
- if (inputColVector.noNulls &&
+ if ((inputColVector.noNulls || !inputColVector.isNull[0]) &&
(myagg.isNull || (inputColVector.compareTo(myagg.value, 0) <OperatorSymbol> 0))) {
myagg.isNull = false;
inputColVector.timestampUpdate(myagg.value, 0);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt
index 3e3d070..c731869 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt
@@ -311,7 +311,7 @@ public class <ClassName> extends VectorAggregateExpression {
<ValueType>[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
if (myagg.isNull) {
myagg.isNull = false;
myagg.sum = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt
index cb9c962..876ead5 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt
@@ -311,7 +311,7 @@ public class <ClassName> extends VectorAggregateExpression {
<ValueType>[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
iterateRepeatingNoNulls(myagg, vector[0], batchSize);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
index 3d03c09..cf19b14 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
@@ -311,7 +311,7 @@ public class <ClassName> extends VectorAggregateExpression {
HiveDecimalWritable[] vector = inputVector.vector;
if (inputVector.isRepeating) {
- if (inputVector.noNulls) {
+ if (inputVector.noNulls || !inputVector.isNull[0]) {
iterateRepeatingNoNulls(myagg, vector[0], inputVector.scale, batchSize);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarMerge.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarMerge.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarMerge.txt
index 570d771..9b1c1cd 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarMerge.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarMerge.txt
@@ -371,7 +371,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputStructColVector.isRepeating) {
- if (inputStructColVector.noNulls) {
+ if (inputStructColVector.noNulls || !inputStructColVector.isNull[0]) {
final long count = countVector[0];
final double sum = sumVector[0];
final double variance = varianceVector[0];
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarTimestamp.txt
----------------------------------------------------------------------
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarTimestamp.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarTimestamp.txt
index d6cd505..1dd5ab4 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarTimestamp.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarTimestamp.txt
@@ -290,7 +290,7 @@ public class <ClassName> extends VectorAggregateExpression {
Aggregation myagg = (Aggregation)agg;
if (inputColVector.isRepeating) {
- if (inputColVector.noNulls) {
+ if (inputColVector.noNulls || !inputColVector.isNull[0]) {
iterateRepeatingNoNulls(myagg, inputColVector.getDouble(0), batchSize);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index 5db3b07..bedc12a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -210,12 +210,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.setElement(outBatchIndex, 0, inColVector);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.setElement(outBatchIndex, inBatchIndex, inColVector);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
@@ -237,12 +239,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.setElement(outBatchIndex, 0, inColVector);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.setElement(outBatchIndex, inBatchIndex, inColVector);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
index 5d1db4d..6ae6727 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
@@ -150,7 +150,7 @@ public class VectorGroupKeyHelper extends VectorColumnSetInfo {
TimestampColumnVector inputColumnVector = (TimestampColumnVector) inputBatch.cols[inputColumnNum];
TimestampColumnVector outputColumnVector = (TimestampColumnVector) outputBatch.cols[outputColumnNum];
if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) {
-
+ outputColumnVector.isNull[outputBatch.size] = false;
outputColumnVector.setElement(outputBatch.size, 0, inputColumnVector);
} else {
outputColumnVector.noNulls = false;
@@ -163,7 +163,7 @@ public class VectorGroupKeyHelper extends VectorColumnSetInfo {
IntervalDayTimeColumnVector inputColumnVector = (IntervalDayTimeColumnVector) inputBatch.cols[inputColumnNum];
IntervalDayTimeColumnVector outputColumnVector = (IntervalDayTimeColumnVector) outputBatch.cols[outputColumnNum];
if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) {
-
+ outputColumnVector.isNull[outputBatch.size] = false;
outputColumnVector.setElement(outputBatch.size, 0, inputColumnVector);
} else {
outputColumnVector.noNulls = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index 74ca9d6..2b401ac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -110,7 +110,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignLongNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignLongNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignLongNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignLongNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -132,7 +132,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignDoubleNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignDoubleNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignDoubleNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDoubleNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -154,7 +154,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignStringNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignStringNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignStringNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignStringNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -176,7 +176,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignDecimalNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDecimalNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -198,7 +198,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignTimestampNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignTimestampNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignTimestampNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignTimestampNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -220,7 +220,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignIntervalDayTimeNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignIntervalDayTimeNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -262,7 +262,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignLongNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignLongNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignLongNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignLongNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -291,7 +291,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignDoubleNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignDoubleNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignDoubleNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDoubleNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -320,7 +320,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignStringNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignStringNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignStringNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignStringNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -349,7 +349,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignDecimalNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignDecimalNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -378,7 +378,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignTimestampNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignTimestampNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignTimestampNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignTimestampNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
@@ -407,7 +407,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
} else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
assignIntervalDayTimeNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
+ } else if ((columnVector.noNulls || !columnVector.isNull[0]) && columnVector.isRepeating) {
assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
} else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
assignIntervalDayTimeNullsNoRepeatingNoSelection(keyIndex, i, batch.size, columnVector);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index d1b52c6..4df6e97 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -2942,6 +2942,20 @@ public class VectorizationContext {
childExpr.subList(2, childExpr.size()));
}
+ if (isNullConst(thenDesc) && isNullConst(elseDesc)) {
+
+ // THEN NULL ELSE NULL: An unusual "case", but possible.
+ final int outputColumnNum = ocm.allocateOutputColumn(returnType);
+
+ final VectorExpression resultExpr =
+ new IfExprNullNull(
+ outputColumnNum);
+
+ resultExpr.setOutputTypeInfo(returnType);
+ resultExpr.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE);
+
+ return resultExpr;
+ }
if (isNullConst(thenDesc)) {
final VectorExpression whenExpr = getVectorExpression(whenDesc, mode);
final VectorExpression elseExpr = getVectorExpression(elseDesc, mode);
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 66de847..44b7c95 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -97,19 +97,6 @@ public class VectorizedBatchUtil {
}
/**
- * Iterates thru all the column vectors and sets noNull to
- * specified value.
- *
- * @param batch
- * Batch on which noNull is set
- */
- public static void setNoNullFields(VectorizedRowBatch batch) {
- for (int i = 0; i < batch.numCols; i++) {
- batch.cols[i].noNulls = true;
- }
- }
-
- /**
* Iterates thru all the column vectors and sets repeating to
* specified column.
*
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index 0e703a5..ff55f50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -423,7 +423,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Boolean) value == true ? 1 : 0);
- lcv.isNull[0] = false;
}
}
break;
@@ -436,7 +435,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Byte) value);
- lcv.isNull[0] = false;
}
}
break;
@@ -449,7 +447,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Short) value);
- lcv.isNull[0] = false;
}
}
break;
@@ -462,7 +459,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Integer) value);
- lcv.isNull[0] = false;
}
}
break;
@@ -475,7 +471,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Long) value);
- lcv.isNull[0] = false;
}
}
break;
@@ -488,7 +483,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill(DateWritable.dateToDays((Date) value));
- lcv.isNull[0] = false;
}
}
break;
@@ -501,7 +495,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill((Timestamp) value);
- lcv.isNull[0] = false;
}
}
break;
@@ -514,7 +507,6 @@ public class VectorizedRowBatchCtx {
lcv.isRepeating = true;
} else {
lcv.fill(((HiveIntervalYearMonth) value).getTotalMonths());
- lcv.isNull[0] = false;
}
}
@@ -526,7 +518,6 @@ public class VectorizedRowBatchCtx {
icv.isRepeating = true;
} else {
icv.fill(((HiveIntervalDayTime) value));
- icv.isNull[0] = false;
}
}
@@ -538,7 +529,6 @@ public class VectorizedRowBatchCtx {
dcv.isRepeating = true;
} else {
dcv.fill((Float) value);
- dcv.isNull[0] = false;
}
}
break;
@@ -551,7 +541,6 @@ public class VectorizedRowBatchCtx {
dcv.isRepeating = true;
} else {
dcv.fill((Double) value);
- dcv.isNull[0] = false;
}
}
break;
@@ -563,10 +552,7 @@ public class VectorizedRowBatchCtx {
dv.isNull[0] = true;
dv.isRepeating = true;
} else {
- HiveDecimal hd = (HiveDecimal) value;
- dv.set(0, hd);
- dv.isRepeating = true;
- dv.isNull[0] = false;
+ dv.fill((HiveDecimal) value);
}
}
break;
@@ -580,7 +566,6 @@ public class VectorizedRowBatchCtx {
bcv.isRepeating = true;
} else {
bcv.fill(bytes);
- bcv.isNull[0] = false;
}
}
break;
@@ -595,8 +580,7 @@ public class VectorizedRowBatchCtx {
bcv.isNull[0] = true;
bcv.isRepeating = true;
} else {
- bcv.setVal(0, sVal.getBytes());
- bcv.isRepeating = true;
+ bcv.fill(sVal.getBytes());
}
}
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/a4689020/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToTimestamp.java
index c15bdc1..f1a584e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDateToTimestamp.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
@@ -58,7 +60,6 @@ public class CastDateToTimestamp extends VectorExpression {
int[] sel = batch.selected;
boolean[] inputIsNull = inputColVector.isNull;
boolean[] outputIsNull = outputColVector.isNull;
- outputColVector.noNulls = inputColVector.noNulls;
int n = batch.size;
long[] vector = inputColVector.vector;
@@ -67,39 +68,73 @@ public class CastDateToTimestamp extends VectorExpression {
return;
}
+ // We do not need to do a column reset since we are carefully changing the output.
+ outputColVector.isRepeating = false;
+
if (inputColVector.isRepeating) {
- //All must be selected otherwise size would be zero
- //Repeating property will not change.
- setDays(outputColVector, vector, 0);
- // Even if there are no nulls, we always copy over entry 0. Simplifies code.
- outputIsNull[0] = inputIsNull[0];
+ if (inputColVector.noNulls || !inputIsNull[0]) {
+ // Set isNull before call in case it changes it mind.
+ outputIsNull[0] = false;
+ setDays(outputColVector, vector, 0);
+ } else {
+ outputIsNull[0] = true;
+ outputColVector.noNulls = false;
+ }
outputColVector.isRepeating = true;
- } else if (inputColVector.noNulls) {
+ return;
+ }
+
+ if (inputColVector.noNulls) {
if (batch.selectedInUse) {
- for(int j = 0; j != n; j++) {
- int i = sel[j];
- setDays(outputColVector, vector, i);
+
+ // CONSIDER: For large n, fill n or all of isNull array and use the tighter ELSE loop.
+
+ if (!outputColVector.noNulls) {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ outputIsNull[i] = false;
+ // Set isNull before call in case it changes it mind.
+ setDays(outputColVector, vector, i);
+ }
+ } else {
+ for(int j = 0; j != n; j++) {
+ final int i = sel[j];
+ // Set isNull before call in case it changes it mind.
+ setDays(outputColVector, vector, i);
+ }
}
} else {
+ if (!outputColVector.noNulls) {
+
+ // Assume it is almost always a performance win to fill all of isNull so we can
+ // safely reset noNulls.
+ Arrays.fill(outputIsNull, false);
+ outputColVector.noNulls = true;
+ }
for(int i = 0; i != n; i++) {
+ // Set isNull before call in case it changes it mind.
setDays(outputColVector, vector, i);
}
}
- outputColVector.isRepeating = false;
- } else /* there are nulls */ {
+ } else /* there are NULLs in the inputColVector */ {
+
+ // Carefully handle NULLs...
+ outputColVector.noNulls = false;
+
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- setDays(outputColVector, vector, i);
+ // Set isNull before call in case it changes it mind.
outputIsNull[i] = inputIsNull[i];
+ setDays(outputColVector, vector, i);
}
} else {
+ // Set isNull before calls in case tney change their mind.
+ System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
for(int i = 0; i != n; i++) {
setDays(outputColVector, vector, i);
}
- System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
- outputColVector.isRepeating = false;
}
}