You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/07/11 08:09:59 UTC
[pinot] branch master updated: Rework transform functions NULL support. (#11008)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9bb3cbb3d6 Rework transform functions NULL support. (#11008)
9bb3cbb3d6 is described below
commit 9bb3cbb3d6ebf5a9fcd8a5410a5794aee02c7ee4
Author: Shen Yu <sh...@startree.ai>
AuthorDate: Tue Jul 11 01:09:54 2023 -0700
Rework transform functions NULL support. (#11008)
---
.../transform/function/BaseTransformFunction.java | 606 +--------------------
.../transform/function/CaseTransformFunction.java | 157 ++----
...lyWhenNullHandlingEnabledTransformFunction.java | 177 ++++++
.../function/GreatestTransformFunction.java | 279 +---------
.../transform/function/LeastTransformFunction.java | 272 +--------
.../function/ScalarTransformFunctionWrapper.java | 410 +-------------
.../SelectTupleElementTransformFunction.java | 290 +++++++++-
.../transform/function/TransformFunction.java | 123 +----
.../function/TransformFunctionFactory.java | 26 +-
.../function/ArrayBaseTransformFunctionTest.java | 57 +-
.../function/BaseTransformFunctionTest.java | 137 ++---
.../function/DateTimeTransformFunctionTest.java | 8 +-
.../function/ExtractTransformFunctionTest.java | 8 +-
.../function/IdentifierTransformFunctionTest.java | 5 +-
.../function/InTransformFunctionTest.java | 15 +-
.../function/LiteralTransformFunctionTest.java | 6 +-
.../TupleSelectionTransformFunctionsTest.java | 94 ++--
.../queries/NullHandlingEnabledQueriesTest.java | 18 +
18 files changed, 751 insertions(+), 1937 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
index 84364a982f..caa97f60bc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
@@ -23,8 +23,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
@@ -102,6 +100,7 @@ public abstract class BaseTransformFunction implements TransformFunction {
protected byte[][][] _bytesValuesMV;
protected List<TransformFunction> _arguments;
+ protected boolean _nullHandlingEnabled;
protected void fillResultUnknown(int length) {
for (int i = 0; i < length; i++) {
@@ -115,6 +114,13 @@ public abstract class BaseTransformFunction implements TransformFunction {
_arguments = arguments;
}
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap,
+ boolean nullHandlingEnabled) {
+ init(arguments, columnContextMap);
+ _nullHandlingEnabled = nullHandlingEnabled;
+ }
+
@Override
public Dictionary getDictionary() {
return null;
@@ -188,56 +194,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _intValuesSV;
}
- @Override
- public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initIntValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- _intValuesSV = transformToIntValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformToLongValuesSVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _intValuesSV, length);
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformToFloatValuesSVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _intValuesSV, length);
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformToDoubleValuesSVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _intValuesSV, length);
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _intValuesSV, length);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _intValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _intValuesSV[i] = (int) DataSchema.ColumnDataType.INT.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as INT", resultDataType));
- }
- return ImmutablePair.of(_intValuesSV, bitmap);
- }
-
protected void initLongValuesSV(int length) {
if (_longValuesSV == null || _longValuesSV.length < length) {
_longValuesSV = new long[length];
@@ -288,56 +244,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _longValuesSV;
}
- @Override
- public Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initLongValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[], RoaringBitmap> intResults = transformToIntValuesSVWithNull(valueBlock);
- bitmap = intResults.getRight();
- ArrayCopyUtils.copy(intResults.getLeft(), _longValuesSV, length);
- break;
- case LONG:
- _longValuesSV = transformToLongValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformToFloatValuesSVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _longValuesSV, length);
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformToDoubleValuesSVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _longValuesSV, length);
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _longValuesSV, length);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _longValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _longValuesSV[i] = (long) DataSchema.ColumnDataType.LONG.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as LONG", resultDataType));
- }
- return ImmutablePair.of(_longValuesSV, bitmap);
- }
-
protected void initFloatValuesSV(int length) {
if (_floatValuesSV == null || _floatValuesSV.length < length) {
_floatValuesSV = new float[length];
@@ -388,56 +294,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _floatValuesSV;
}
- @Override
- public Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initFloatValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[], RoaringBitmap> intResult = transformToIntValuesSVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _floatValuesSV, length);
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformToLongValuesSVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _floatValuesSV, length);
- break;
- case FLOAT:
- _floatValuesSV = transformToFloatValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformToDoubleValuesSVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _floatValuesSV, length);
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _floatValuesSV, length);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _floatValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _floatValuesSV[i] = (float) DataSchema.ColumnDataType.FLOAT.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as FLOAT", resultDataType));
- }
- return ImmutablePair.of(_floatValuesSV, bitmap);
- }
-
protected void initDoubleValuesSV(int length) {
if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
_doubleValuesSV = new double[length];
@@ -488,56 +344,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _doubleValuesSV;
}
- @Override
- public Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initDoubleValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[], RoaringBitmap> intResult = transformToIntValuesSVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _doubleValuesSV, length);
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformToLongValuesSVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _doubleValuesSV, length);
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformToFloatValuesSVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _doubleValuesSV, length);
- break;
- case DOUBLE:
- _doubleValuesSV = transformToDoubleValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _doubleValuesSV, length);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _doubleValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _doubleValuesSV[i] = (double) DataSchema.ColumnDataType.DOUBLE.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as DOUBLE", resultDataType));
- }
- return ImmutablePair.of(_doubleValuesSV, bitmap);
- }
-
protected void initBigDecimalValuesSV(int length) {
if (_bigDecimalValuesSV == null || _bigDecimalValuesSV.length < length) {
_bigDecimalValuesSV = new BigDecimal[length];
@@ -592,61 +398,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _bigDecimalValuesSV;
}
- @Override
- public Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initBigDecimalValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[], RoaringBitmap> intResult = transformToIntValuesSVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformToLongValuesSVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformToFloatValuesSVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformToDoubleValuesSVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case BIG_DECIMAL:
- _bigDecimalValuesSV = transformToBigDecimalValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case BYTES:
- Pair<byte[][], RoaringBitmap> byteResult = transformToBytesValuesSVWithNull(valueBlock);
- bitmap = byteResult.getRight();
- ArrayCopyUtils.copy(byteResult.getLeft(), _bigDecimalValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _bigDecimalValuesSV[i] = (BigDecimal) DataSchema.ColumnDataType.BIG_DECIMAL.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as BIG_DECIMAL", resultDataType));
- }
- return ImmutablePair.of(_bigDecimalValuesSV, bitmap);
- }
-
protected void initStringValuesSV(int length) {
if (_stringValuesSV == null || _stringValuesSV.length < length) {
_stringValuesSV = new String[length];
@@ -701,61 +452,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _stringValuesSV;
}
- @Override
- public Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initStringValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[], RoaringBitmap> intResult = transformToIntValuesSVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _stringValuesSV, length);
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformToLongValuesSVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _stringValuesSV, length);
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformToFloatValuesSVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _stringValuesSV, length);
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformToDoubleValuesSVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _stringValuesSV, length);
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _stringValuesSV, length);
- break;
- case STRING:
- _stringValuesSV = transformToStringValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case BYTES:
- Pair<byte[][], RoaringBitmap> byteResult = transformToBytesValuesSVWithNull(valueBlock);
- bitmap = byteResult.getRight();
- ArrayCopyUtils.copy(byteResult.getLeft(), _stringValuesSV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _stringValuesSV[i] = (String) DataSchema.ColumnDataType.STRING.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as STRING", resultDataType));
- }
- return ImmutablePair.of(_stringValuesSV, bitmap);
- }
-
protected void initBytesValuesSV(int length) {
if (_bytesValuesSV == null || _bytesValuesSV.length < length) {
_bytesValuesSV = new byte[length][];
@@ -794,41 +490,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _bytesValuesSV;
}
- @Override
- public Pair<byte[][], RoaringBitmap> transformToBytesValuesSVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initBytesValuesSV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult = transformToBigDecimalValuesSVWithNull(valueBlock);
- bitmap = bigDecimalResult.getRight();
- ArrayCopyUtils.copy(bigDecimalResult.getLeft(), _bytesValuesSV, length);
- break;
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformToStringValuesSVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _bytesValuesSV, length);
- break;
- case BYTES:
- _bytesValuesSV = transformToBytesValuesSV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case UNKNOWN:
- // Copy the values to ensure behaviour consistency with non null-handling.
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- for (int i = 0; i < length; i++) {
- _bytesValuesSV[i] = (byte[]) DataSchema.ColumnDataType.BYTES.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read SV %s as BYTES", resultDataType));
- }
- return ImmutablePair.of(_bytesValuesSV, bitmap);
- }
-
protected void initIntValuesMV(int length) {
if (_intValuesMV == null || _intValuesMV.length < length) {
_intValuesMV = new int[length][];
@@ -881,51 +542,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _intValuesMV;
}
- @Override
- public Pair<int[][], RoaringBitmap> transformToIntValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initIntValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- _intValuesMV = transformToIntValuesMV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case LONG:
- Pair<long[][], RoaringBitmap> longResult = transformToLongValuesMVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _intValuesMV, length);
- break;
- case FLOAT:
- Pair<float[][], RoaringBitmap> floatResult = transformToFloatValuesMVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _intValuesMV, length);
- break;
- case DOUBLE:
- Pair<double[][], RoaringBitmap> doubleResult = transformToDoubleValuesMVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _intValuesMV, length);
- break;
- case STRING:
- Pair<String[][], RoaringBitmap> stringResult = transformToStringValuesMVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _intValuesMV, length);
- break;
- case UNKNOWN:
- // Copy the values to ensure behaviour consistency with non null-handling.
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- for (int i = 0; i < length; i++) {
- _intValuesMV[i] = (int[]) DataSchema.ColumnDataType.INT_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as INT", resultDataType));
- }
- return ImmutablePair.of(_intValuesMV, bitmap);
- }
-
protected void initLongValuesMV(int length) {
if (_longValuesMV == null || _longValuesMV.length < length) {
_longValuesMV = new long[length][];
@@ -978,47 +594,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _longValuesMV;
}
- @Override
- public Pair<long[][], RoaringBitmap> transformToLongValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initLongValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[][], RoaringBitmap> intResult = transformToIntValuesMVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _longValuesMV, length);
- break;
- case FLOAT:
- Pair<float[][], RoaringBitmap> floatResult = transformToFloatValuesMVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _longValuesMV, length);
- break;
- case DOUBLE:
- Pair<double[][], RoaringBitmap> doubleResult = transformToDoubleValuesMVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _longValuesMV, length);
- break;
- case STRING:
- Pair<String[][], RoaringBitmap> stringResult = transformToStringValuesMVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _longValuesMV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _longValuesMV[i] = (long[]) DataSchema.ColumnDataType.LONG_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as LONG", resultDataType));
- }
- return ImmutablePair.of(_longValuesMV, bitmap);
- }
-
protected void initFloatValuesMV(int length) {
if (_floatValuesMV == null || _floatValuesMV.length < length) {
_floatValuesMV = new float[length][];
@@ -1071,51 +646,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _floatValuesMV;
}
- @Override
- public Pair<float[][], RoaringBitmap> transformToFloatValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initFloatValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[][], RoaringBitmap> intResult = transformToIntValuesMVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _floatValuesMV, length);
- break;
- case LONG:
- Pair<long[][], RoaringBitmap> longResult = transformToLongValuesMVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _floatValuesMV, length);
- break;
- case FLOAT:
- _floatValuesMV = transformToFloatValuesMV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case DOUBLE:
- Pair<double[][], RoaringBitmap> doubleResult = transformToDoubleValuesMVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _floatValuesMV, length);
- break;
- case STRING:
- Pair<String[][], RoaringBitmap> stringResult = transformToStringValuesMVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _floatValuesMV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _floatValuesMV[i] = (float[]) DataSchema.ColumnDataType.FLOAT_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as FLOAT", resultDataType));
- }
- return ImmutablePair.of(_floatValuesMV, bitmap);
- }
-
protected void initDoubleValuesMV(int length) {
if (_doubleValuesMV == null || _doubleValuesMV.length < length) {
_doubleValuesMV = new double[length][];
@@ -1168,51 +698,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _doubleValuesMV;
}
- @Override
- public Pair<double[][], RoaringBitmap> transformToDoubleValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initDoubleValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType.getStoredType()) {
- case INT:
- Pair<int[][], RoaringBitmap> intResult = transformToIntValuesMVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _doubleValuesMV, length);
- break;
- case LONG:
- Pair<long[][], RoaringBitmap> longResult = transformToLongValuesMVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _doubleValuesMV, length);
- break;
- case FLOAT:
- Pair<float[][], RoaringBitmap> floatResult = transformToFloatValuesMVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _doubleValuesMV, length);
- break;
- case DOUBLE:
- _doubleValuesMV = transformToDoubleValuesMV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case STRING:
- Pair<String[][], RoaringBitmap> stringResult = transformToStringValuesMVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _doubleValuesMV, length);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _doubleValuesMV[i] = (double[]) DataSchema.ColumnDataType.DOUBLE_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as DOUBLE", resultDataType));
- }
- return ImmutablePair.of(_doubleValuesMV, bitmap);
- }
-
protected void initStringValuesMV(int length) {
if (_stringValuesMV == null || _stringValuesMV.length < length) {
_stringValuesMV = new String[length][];
@@ -1265,51 +750,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _stringValuesMV;
}
- @Override
- public Pair<String[][], RoaringBitmap> transformToStringValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initStringValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType) {
- case INT:
- Pair<int[][], RoaringBitmap> intResult = transformToIntValuesMVWithNull(valueBlock);
- bitmap = intResult.getRight();
- ArrayCopyUtils.copy(intResult.getLeft(), _stringValuesMV, length);
- break;
- case LONG:
- Pair<long[][], RoaringBitmap> longResult = transformToLongValuesMVWithNull(valueBlock);
- bitmap = longResult.getRight();
- ArrayCopyUtils.copy(longResult.getLeft(), _stringValuesMV, length);
- break;
- case FLOAT:
- Pair<float[][], RoaringBitmap> floatResult = transformToFloatValuesMVWithNull(valueBlock);
- bitmap = floatResult.getRight();
- ArrayCopyUtils.copy(floatResult.getLeft(), _stringValuesMV, length);
- break;
- case DOUBLE:
- Pair<double[][], RoaringBitmap> doubleResult = transformToDoubleValuesMVWithNull(valueBlock);
- bitmap = doubleResult.getRight();
- ArrayCopyUtils.copy(doubleResult.getLeft(), _stringValuesMV, length);
- break;
- case STRING:
- _stringValuesMV = transformToStringValuesMV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _stringValuesMV[i] = (String[]) DataSchema.ColumnDataType.STRING_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as STRING", resultDataType));
- }
- return ImmutablePair.of(_stringValuesMV, bitmap);
- }
-
protected void initBytesValuesMV(int length) {
if (_bytesValuesMV == null || _bytesValuesMV.length < length) {
_bytesValuesMV = new byte[length][][];
@@ -1338,36 +778,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _bytesValuesMV;
}
- @Override
- public Pair<byte[][][], RoaringBitmap> transformToBytesValuesMVWithNull(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- initBytesValuesMV(length);
- RoaringBitmap bitmap;
- DataType resultDataType = getResultMetadata().getDataType();
- switch (resultDataType) {
- case STRING:
- Pair<String[][], RoaringBitmap> stringResult = transformToStringValuesMVWithNull(valueBlock);
- bitmap = stringResult.getRight();
- ArrayCopyUtils.copy(stringResult.getLeft(), _bytesValuesMV, length);
- break;
- case BYTES:
- _bytesValuesMV = transformToBytesValuesMV(valueBlock);
- bitmap = getNullBitmap(valueBlock);
- break;
- case UNKNOWN:
- bitmap = new RoaringBitmap();
- bitmap.add(0L, length);
- // Copy the values to ensure behaviour consistency with non null-handling.
- for (int i = 0; i < length; i++) {
- _bytesValuesMV[i] = (byte[][]) DataSchema.ColumnDataType.BYTES_ARRAY.getNullPlaceholder();
- }
- break;
- default:
- throw new IllegalStateException(String.format("Cannot read MV %s as bytes", resultDataType));
- }
- return ImmutablePair.of(_bytesValuesMV, bitmap);
- }
-
@Nullable
@Override
public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java
index 58d97d069a..f4ff9330b3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java
@@ -54,7 +54,7 @@ import org.roaringbitmap.RoaringBitmap;
* ELSE_EXPRESSION can be omitted. When none of when statements is evaluated to be true, and there is no else
* expression, we output null. Note that when statement is considered as false if it is evaluated to be null.
*/
-public class CaseTransformFunction extends BaseTransformFunction {
+public class CaseTransformFunction extends ComputeDifferentlyWhenNullHandlingEnabledTransformFunction {
public static final String FUNCTION_NAME = "case";
private List<TransformFunction> _whenStatements = new ArrayList<>();
@@ -71,7 +71,9 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap) {
+ public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap,
+ boolean nullHandlingEnabled) {
+ super.init(arguments, columnContextMap, nullHandlingEnabled);
// Check that there are more than 2 arguments
// Else statement can be omitted.
if (arguments.size() < 2) {
@@ -298,9 +300,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
if (!nullHandlingEnabled) {
return whenStatement.transformToIntValuesSV(valueBlock);
}
- Pair<int[], RoaringBitmap> result = whenStatement.transformToIntValuesSVWithNull(valueBlock);
- RoaringBitmap bitmap = result.getRight();
- int[] intResult = result.getLeft();
+ int[] intResult = whenStatement.transformToIntValuesSV(valueBlock);
+ RoaringBitmap bitmap = whenStatement.getNullBitmap(valueBlock);
if (bitmap != null) {
for (int i : bitmap) {
intResult[i] = 0;
@@ -310,10 +311,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public int[] transformToIntValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
- return super.transformToIntValuesSV(valueBlock);
- }
+ protected int[] transformToIntValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initIntValuesSV(numDocs);
@@ -348,10 +346,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
- return super.transformToIntValuesSVWithNull(valueBlock);
- }
+ protected int[] transformToIntValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -362,7 +357,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<int[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToIntValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i, ImmutablePair.of(_thenStatements.get(i).transformToIntValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -386,9 +382,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<int[], RoaringBitmap> intValuesNullPair = _elseStatement.transformToIntValuesSVWithNull(valueBlock);
- int[] intValues = intValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = intValuesNullPair.getRight();
+ int[] intValues = _elseStatement.transformToIntValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_intValuesSV[docId] = intValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -397,14 +392,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_intValuesSV, bitmap);
+ return _intValuesSV;
}
@Override
- public long[] transformToLongValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
- return super.transformToLongValuesSV(valueBlock);
- }
+ protected long[] transformToLongValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initLongValuesSV(numDocs);
@@ -439,10 +431,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
- return super.transformToLongValuesSVWithNull(valueBlock);
- }
+ protected long[] transformToLongValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -453,7 +442,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<long[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToLongValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i, ImmutablePair.of(_thenStatements.get(i).transformToLongValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -477,9 +467,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<long[], RoaringBitmap> longValuesNullPair = _elseStatement.transformToLongValuesSVWithNull(valueBlock);
- long[] longValues = longValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = longValuesNullPair.getRight();
+ long[] longValues = _elseStatement.transformToLongValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_longValuesSV[docId] = longValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -488,14 +477,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_longValuesSV, bitmap);
+ return _longValuesSV;
}
@Override
- public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
- return super.transformToFloatValuesSV(valueBlock);
- }
+ protected float[] transformToFloatValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initFloatValuesSV(numDocs);
@@ -530,10 +516,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
- return super.transformToFloatValuesSVWithNull(valueBlock);
- }
+ protected float[] transformToFloatValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -544,7 +527,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<float[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToFloatValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i, ImmutablePair.of(_thenStatements.get(i).transformToFloatValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -568,9 +552,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<float[], RoaringBitmap> floatValuesNullPair = _elseStatement.transformToFloatValuesSVWithNull(valueBlock);
- float[] floatValues = floatValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = floatValuesNullPair.getRight();
+ float[] floatValues = _elseStatement.transformToFloatValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_floatValuesSV[docId] = floatValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -579,14 +562,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_floatValuesSV, bitmap);
+ return _floatValuesSV;
}
@Override
- public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
- return super.transformToDoubleValuesSV(valueBlock);
- }
+ protected double[] transformToDoubleValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initDoubleValuesSV(numDocs);
@@ -621,10 +601,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
- return super.transformToDoubleValuesSVWithNull(valueBlock);
- }
+ protected double[] transformToDoubleValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -635,7 +612,9 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<double[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToDoubleValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i,
+ ImmutablePair.of(_thenStatements.get(i).transformToDoubleValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -659,10 +638,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<double[], RoaringBitmap> doubleValuesNullPair =
- _elseStatement.transformToDoubleValuesSVWithNull(valueBlock);
- double[] doubleValues = doubleValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = doubleValuesNullPair.getRight();
+ double[] doubleValues = _elseStatement.transformToDoubleValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_doubleValuesSV[docId] = doubleValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -671,14 +648,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_doubleValuesSV, bitmap);
+ return _doubleValuesSV;
}
@Override
- public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BIG_DECIMAL) {
- return super.transformToBigDecimalValuesSV(valueBlock);
- }
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initBigDecimalValuesSV(numDocs);
@@ -713,10 +687,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BIG_DECIMAL) {
- return super.transformToBigDecimalValuesSVWithNull(valueBlock);
- }
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -727,7 +698,9 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<BigDecimal[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToBigDecimalValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i,
+ ImmutablePair.of(_thenStatements.get(i).transformToBigDecimalValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -751,10 +724,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<BigDecimal[], RoaringBitmap> bigDecimalValuesNullPair =
- _elseStatement.transformToBigDecimalValuesSVWithNull(valueBlock);
- BigDecimal[] bigDecimalValues = bigDecimalValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = bigDecimalValuesNullPair.getRight();
+ BigDecimal[] bigDecimalValues = _elseStatement.transformToBigDecimalValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_bigDecimalValuesSV[docId] = bigDecimalValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -763,14 +734,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_bigDecimalValuesSV, bitmap);
+ return _bigDecimalValuesSV;
}
@Override
- public String[] transformToStringValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
- return super.transformToStringValuesSV(valueBlock);
- }
+ protected String[] transformToStringValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initStringValuesSV(numDocs);
@@ -805,10 +773,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
- return super.transformToStringValuesSVWithNull(valueBlock);
- }
+ protected String[] transformToStringValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -819,7 +784,9 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<String[], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToStringValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i,
+ ImmutablePair.of(_thenStatements.get(i).transformToStringValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -843,10 +810,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<String[], RoaringBitmap> stringValuesNullPair =
- _elseStatement.transformToStringValuesSVWithNull(valueBlock);
- String[] stringValues = stringValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = stringValuesNullPair.getRight();
+ String[] stringValues = _elseStatement.transformToStringValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_stringValuesSV[docId] = stringValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -855,14 +820,11 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_stringValuesSV, bitmap);
+ return _stringValuesSV;
}
@Override
- public byte[][] transformToBytesValuesSV(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) {
- return super.transformToBytesValuesSV(valueBlock);
- }
+ protected byte[][] transformToBytesValuesSVUsingValue(ValueBlock valueBlock) {
int[] selected = getSelectedArray(valueBlock, false);
int numDocs = valueBlock.getNumDocs();
initBytesValuesSV(numDocs);
@@ -897,10 +859,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
@Override
- public Pair<byte[][], RoaringBitmap> transformToBytesValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) {
- return super.transformToBytesValuesSVWithNull(valueBlock);
- }
+ protected byte[][] transformToBytesValuesSVUsingValueAndNull(ValueBlock valueBlock) {
final RoaringBitmap bitmap = new RoaringBitmap();
int[] selected = getSelectedArray(valueBlock, true);
int numDocs = valueBlock.getNumDocs();
@@ -911,7 +870,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
Map<Integer, Pair<byte[][], RoaringBitmap>> thenStatementsIndexToValues = new HashMap<>();
for (int i = 0; i < numThenStatements; i++) {
if (_computeThenStatements[i]) {
- thenStatementsIndexToValues.put(i, _thenStatements.get(i).transformToBytesValuesSVWithNull(valueBlock));
+ thenStatementsIndexToValues.put(i, ImmutablePair.of(_thenStatements.get(i).transformToBytesValuesSV(valueBlock),
+ _thenStatements.get(i).getNullBitmap(valueBlock)));
}
}
for (int docId = 0; docId < numDocs; docId++) {
@@ -935,9 +895,8 @@ public class CaseTransformFunction extends BaseTransformFunction {
bitmap.add(docId);
}
} else {
- Pair<byte[][], RoaringBitmap> bytesValuesNullPair = _elseStatement.transformToBytesValuesSVWithNull(valueBlock);
- byte[][] byteValues = bytesValuesNullPair.getLeft();
- RoaringBitmap nullBitmap = bytesValuesNullPair.getRight();
+ byte[][] byteValues = _elseStatement.transformToBytesValuesSV(valueBlock);
+ RoaringBitmap nullBitmap = _elseStatement.getNullBitmap(valueBlock);
for (int docId = unselectedDocs.nextSetBit(0); docId >= 0; docId = unselectedDocs.nextSetBit(docId + 1)) {
_bytesValuesSV[docId] = byteValues[docId];
if (nullBitmap != null && nullBitmap.contains(docId)) {
@@ -946,7 +905,7 @@ public class CaseTransformFunction extends BaseTransformFunction {
}
}
}
- return ImmutablePair.of(_bytesValuesSV, bitmap);
+ return _bytesValuesSV;
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ComputeDifferentlyWhenNullHandlingEnabledTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ComputeDifferentlyWhenNullHandlingEnabledTransformFunction.java
new file mode 100644
index 0000000000..5d2d6b8da6
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ComputeDifferentlyWhenNullHandlingEnabledTransformFunction.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.math.BigDecimal;
+import javax.annotation.Nullable;
+import org.apache.pinot.core.operator.blocks.ValueBlock;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.roaringbitmap.RoaringBitmap;
+
+
+/**
+ * Base class for transform functions that compute differently (using value and NULL together) when NULL handling is
+ * enabled.
+ */
+public abstract class ComputeDifferentlyWhenNullHandlingEnabledTransformFunction extends BaseTransformFunction {
+
+ @Override
+ public int[] transformToIntValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.INT) {
+ return super.transformToIntValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToIntValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToIntValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected int[] transformToIntValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected int[] transformToIntValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long[] transformToLongValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.LONG) {
+ return super.transformToLongValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToLongValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToLongValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected long[] transformToLongValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected long[] transformToLongValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.FLOAT) {
+ return super.transformToFloatValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToFloatValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToFloatValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected float[] transformToFloatValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected float[] transformToFloatValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.DOUBLE) {
+ return super.transformToDoubleValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToDoubleValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToDoubleValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected double[] transformToDoubleValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected double[] transformToDoubleValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.BIG_DECIMAL) {
+ return super.transformToBigDecimalValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToBigDecimalValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToBigDecimalValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String[] transformToStringValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.STRING) {
+ return super.transformToStringValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToStringValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToStringValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected String[] transformToStringValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected String[] transformToStringValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[][] transformToBytesValuesSV(ValueBlock valueBlock) {
+ if (getResultMetadata().getDataType().getStoredType() != FieldSpec.DataType.BYTES) {
+ return super.transformToBytesValuesSV(valueBlock);
+ }
+ if (_nullHandlingEnabled) {
+ return transformToBytesValuesSVUsingValueAndNull(valueBlock);
+ } else {
+ return transformToBytesValuesSVUsingValue(valueBlock);
+ }
+ }
+
+ protected byte[][] transformToBytesValuesSVUsingValue(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ protected byte[][] transformToBytesValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Nullable
+ @Override
+ public abstract RoaringBitmap getNullBitmap(ValueBlock valueBlock);
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
index 32217bd40d..686a8d4f77 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GreatestTransformFunction.java
@@ -16,14 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
-import org.apache.pinot.core.operator.blocks.ValueBlock;
-import org.roaringbitmap.RoaringBitmap;
/**
* The <code>GreatestTransformFunction</code> implements the Greatest operator.
@@ -42,280 +39,36 @@ public class GreatestTransformFunction extends SelectTupleElementTransformFuncti
}
@Override
- public int[] transformToIntValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initIntValuesSV(numDocs);
- int[] values = _arguments.get(0).transformToIntValuesSV(valueBlock);
- System.arraycopy(values, 0, _intValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToIntValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _intValuesSV[j] = Math.max(_intValuesSV[j], values[j]);
- }
- }
- return _intValuesSV;
- }
-
- @Override
- public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initIntValuesSV(numDocs);
- Pair<int[], RoaringBitmap> values = _arguments.get(0).transformToIntValuesSVWithNull(valueBlock);
- int[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _intValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToIntValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _intValuesSV[j] = values.getLeft()[j];
- } else {
- _intValuesSV[j] = Math.max(_intValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_intValuesSV, nullBitmap);
- }
-
- @Override
- public long[] transformToLongValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initLongValuesSV(numDocs);
- long[] values = _arguments.get(0).transformToLongValuesSV(valueBlock);
- System.arraycopy(values, 0, _longValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToLongValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _longValuesSV[j] = Math.max(_longValuesSV[j], values[j]);
- }
- }
- return _longValuesSV;
- }
-
- @Override
- public Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initLongValuesSV(numDocs);
- Pair<long[], RoaringBitmap> values = _arguments.get(0).transformToLongValuesSVWithNull(valueBlock);
- long[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _longValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToLongValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _longValuesSV[j] = values.getLeft()[j];
- } else {
- _longValuesSV[j] = Math.max(_longValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_longValuesSV, nullBitmap);
- }
-
- @Override
- public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initFloatValuesSV(numDocs);
- float[] values = _arguments.get(0).transformToFloatValuesSV(valueBlock);
- System.arraycopy(values, 0, _floatValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToFloatValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _floatValuesSV[j] = Math.max(_floatValuesSV[j], values[j]);
- }
- }
- return _floatValuesSV;
+ protected int binaryFunction(int a, int b) {
+ return Math.max(a, b);
}
@Override
- public Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initFloatValuesSV(numDocs);
- Pair<float[], RoaringBitmap> values = _arguments.get(0).transformToFloatValuesSVWithNull(valueBlock);
- float[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _floatValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToFloatValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull != null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _floatValuesSV[j] = values.getLeft()[j];
- } else {
- _floatValuesSV[j] = Math.max(_floatValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_floatValuesSV, nullBitmap);
+ protected long binaryFunction(long a, long b) {
+ return Math.max(a, b);
}
@Override
- public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initDoubleValuesSV(numDocs);
- double[] values = _arguments.get(0).transformToDoubleValuesSV(valueBlock);
- System.arraycopy(values, 0, _doubleValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToDoubleValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _doubleValuesSV[j] = Math.max(_doubleValuesSV[j], values[j]);
- }
- }
- return _doubleValuesSV;
+ protected float binaryFunction(float a, float b) {
+ return Math.max(a, b);
}
@Override
- public Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initDoubleValuesSV(numDocs);
- Pair<double[], RoaringBitmap> values = _arguments.get(0).transformToDoubleValuesSVWithNull(valueBlock);
- double[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _doubleValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToDoubleValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _doubleValuesSV[j] = values.getLeft()[j];
- } else {
- _doubleValuesSV[j] = Math.max(_doubleValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_doubleValuesSV, nullBitmap);
+ protected double binaryFunction(double a, double b) {
+ return Math.max(a, b);
}
@Override
- public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initBigDecimalValuesSV(numDocs);
- BigDecimal[] values = _arguments.get(0).transformToBigDecimalValuesSV(valueBlock);
- System.arraycopy(values, 0, _bigDecimalValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToBigDecimalValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _bigDecimalValuesSV[j] = _bigDecimalValuesSV[j].max(values[j]);
- }
- }
- return _bigDecimalValuesSV;
- }
-
- @Override
- public Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initBigDecimalValuesSV(numDocs);
- Pair<BigDecimal[], RoaringBitmap> values = _arguments.get(0).transformToBigDecimalValuesSVWithNull(valueBlock);
- BigDecimal[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _bigDecimalValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToBigDecimalValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _bigDecimalValuesSV[j] = values.getLeft()[j];
- } else {
- _bigDecimalValuesSV[j] = _bigDecimalValuesSV[j].max(values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_bigDecimalValuesSV, nullBitmap);
- }
-
- @Override
- public String[] transformToStringValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initStringValuesSV(numDocs);
- String[] values = _arguments.get(0).transformToStringValuesSV(valueBlock);
- System.arraycopy(values, 0, _stringValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToStringValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- if (_stringValuesSV[j].compareTo(values[j]) < 0) {
- _stringValuesSV[j] = values[j];
- }
- }
- }
- return _stringValuesSV;
+ protected BigDecimal binaryFunction(BigDecimal a, BigDecimal b) {
+ return a.max(b);
}
@Override
- public Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initStringValuesSV(numDocs);
- Pair<String[], RoaringBitmap> values = _arguments.get(0).transformToStringValuesSVWithNull(valueBlock);
- String[] curValues = values.getLeft();
- System.arraycopy(curValues, 0, _stringValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToStringValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing maximum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _stringValuesSV[j] = values.getLeft()[j];
- } else if (_stringValuesSV[j].compareTo(values.getLeft()[j]) < 0) {
- _stringValuesSV[j] = values.getLeft()[j];
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
+ protected String binaryFunction(String a, String b) {
+ if (a.compareTo(b) < 0) {
+ return b;
+ } else {
+ return a;
}
- return ImmutablePair.of(_stringValuesSV, nullBitmap);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
index 4933266f06..0420f2fd1d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LeastTransformFunction.java
@@ -19,11 +19,7 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
-import org.apache.pinot.core.operator.blocks.ValueBlock;
-import org.roaringbitmap.RoaringBitmap;
/**
* The <code>LeastTransformFunction</code> implements the Least operator.
@@ -42,274 +38,36 @@ public class LeastTransformFunction extends SelectTupleElementTransformFunction
}
@Override
- public int[] transformToIntValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initIntValuesSV(numDocs);
- int[] values = _arguments.get(0).transformToIntValuesSV(valueBlock);
- System.arraycopy(values, 0, _intValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToIntValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _intValuesSV[j] = Math.min(_intValuesSV[j], values[j]);
- }
- }
- return _intValuesSV;
- }
-
- @Override
- public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initIntValuesSV(numDocs);
- Pair<int[], RoaringBitmap> values = _arguments.get(0).transformToIntValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _intValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToIntValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _intValuesSV[j] = values.getLeft()[j];
- } else {
- _intValuesSV[j] = Math.min(_intValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_intValuesSV, nullBitmap);
+ protected int binaryFunction(int a, int b) {
+ return Math.min(a, b);
}
@Override
- public long[] transformToLongValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initLongValuesSV(numDocs);
- long[] values = _arguments.get(0).transformToLongValuesSV(valueBlock);
- System.arraycopy(values, 0, _longValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToLongValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _longValuesSV[j] = Math.min(_longValuesSV[j], values[j]);
- }
- }
- return _longValuesSV;
+ protected long binaryFunction(long a, long b) {
+ return Math.min(a, b);
}
@Override
- public Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initLongValuesSV(numDocs);
- Pair<long[], RoaringBitmap> values = _arguments.get(0).transformToLongValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _longValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToLongValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _longValuesSV[j] = values.getLeft()[j];
- } else {
- _longValuesSV[j] = Math.min(_longValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_longValuesSV, nullBitmap);
+ protected float binaryFunction(float a, float b) {
+ return Math.min(a, b);
}
@Override
- public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initFloatValuesSV(numDocs);
- float[] values = _arguments.get(0).transformToFloatValuesSV(valueBlock);
- System.arraycopy(values, 0, _floatValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToFloatValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _floatValuesSV[j] = Math.min(_floatValuesSV[j], values[j]);
- }
- }
- return _floatValuesSV;
+ protected double binaryFunction(double a, double b) {
+ return Math.min(a, b);
}
@Override
- public Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initFloatValuesSV(numDocs);
- Pair<float[], RoaringBitmap> values = _arguments.get(0).transformToFloatValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _floatValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToFloatValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull != null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _floatValuesSV[j] = values.getLeft()[j];
- } else {
- _floatValuesSV[j] = Math.min(_floatValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_floatValuesSV, nullBitmap);
- }
-
- @Override
- public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initDoubleValuesSV(numDocs);
- double[] values = _arguments.get(0).transformToDoubleValuesSV(valueBlock);
- System.arraycopy(values, 0, _doubleValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToDoubleValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _doubleValuesSV[j] = Math.min(_doubleValuesSV[j], values[j]);
- }
- }
- return _doubleValuesSV;
- }
-
- @Override
- public Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initDoubleValuesSV(numDocs);
- Pair<double[], RoaringBitmap> values = _arguments.get(0).transformToDoubleValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _doubleValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToDoubleValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _doubleValuesSV[j] = values.getLeft()[j];
- } else {
- _doubleValuesSV[j] = Math.min(_doubleValuesSV[j], values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_doubleValuesSV, nullBitmap);
- }
-
- @Override
- public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initBigDecimalValuesSV(numDocs);
- BigDecimal[] values = _arguments.get(0).transformToBigDecimalValuesSV(valueBlock);
- System.arraycopy(values, 0, _bigDecimalValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToBigDecimalValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- _bigDecimalValuesSV[j] = _bigDecimalValuesSV[j].min(values[j]);
- }
- }
- return _bigDecimalValuesSV;
- }
-
- @Override
- public Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initBigDecimalValuesSV(numDocs);
- Pair<BigDecimal[], RoaringBitmap> values = _arguments.get(0).transformToBigDecimalValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _bigDecimalValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToBigDecimalValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _bigDecimalValuesSV[j] = values.getLeft()[j];
- } else {
- _bigDecimalValuesSV[j] = _bigDecimalValuesSV[j].min(values.getLeft()[j]);
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
- }
- return ImmutablePair.of(_bigDecimalValuesSV, nullBitmap);
- }
-
- @Override
- public String[] transformToStringValuesSV(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initStringValuesSV(numDocs);
- String[] values = _arguments.get(0).transformToStringValuesSV(valueBlock);
- System.arraycopy(values, 0, _stringValuesSV, 0, numDocs);
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToStringValuesSV(valueBlock);
- for (int j = 0; j < numDocs & j < values.length; j++) {
- if (_stringValuesSV[j].compareTo(values[j]) > 0) {
- _stringValuesSV[j] = values[j];
- }
- }
- }
- return _stringValuesSV;
+ protected BigDecimal binaryFunction(BigDecimal a, BigDecimal b) {
+ return a.min(b);
}
@Override
- public Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- int numDocs = valueBlock.getNumDocs();
- initStringValuesSV(numDocs);
- Pair<String[], RoaringBitmap> values = _arguments.get(0).transformToStringValuesSVWithNull(valueBlock);
- System.arraycopy(values.getLeft(), 0, _stringValuesSV, 0, numDocs);
- RoaringBitmap nullBitmap = values.getRight();
- for (int i = 1; i < _arguments.size(); i++) {
- values = _arguments.get(i).transformToStringValuesSVWithNull(valueBlock);
- RoaringBitmap curNull = values.getRight();
- for (int j = 0; j < numDocs & j < values.getLeft().length; j++) {
- // If current value is not null, we process the data.
- if (curNull == null || !curNull.contains(j)) {
- // If existing minimum value is null, we set the value directly.
- if (nullBitmap != null && nullBitmap.contains(j)) {
- _stringValuesSV[j] = values.getLeft()[j];
- } else if (_stringValuesSV[j].compareTo(values.getLeft()[j]) > 0) {
- _stringValuesSV[j] = values.getLeft()[j];
- }
- }
- }
- if (nullBitmap != null && curNull != null) {
- nullBitmap.and(curNull);
- } else {
- nullBitmap = null;
- }
+ protected String binaryFunction(String a, String b) {
+ if (a.compareTo(b) < 0) {
+ return a;
+ } else {
+ return b;
}
- return ImmutablePair.of(_stringValuesSV, nullBitmap);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
index bf3444a25a..410e979132 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
@@ -24,8 +24,6 @@ import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.FunctionInfo;
import org.apache.pinot.common.function.FunctionInvoker;
import org.apache.pinot.common.function.FunctionUtils;
@@ -34,8 +32,6 @@ import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.spi.data.FieldSpec.DataType;
-import org.roaringbitmap.IntConsumer;
-import org.roaringbitmap.RoaringBitmap;
/**
@@ -83,6 +79,7 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
@Override
public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap) {
+ super.init(arguments, columnContextMap);
int numArguments = arguments.size();
PinotDataType[] parameterTypes = _functionInvoker.getParameterTypes();
Preconditions.checkArgument(numArguments == parameterTypes.length,
@@ -167,29 +164,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _intValuesSV;
}
- @Override
- public Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
- return super.transformToIntValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initIntValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _intValuesSV[i] = (int) result;
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_intValuesSV, bitmap);
- }
-
@Override
public long[] transformToLongValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
@@ -207,29 +181,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _longValuesSV;
}
- @Override
- public Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
- return super.transformToLongValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initLongValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _longValuesSV[i] = (long) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_longValuesSV, bitmap);
- }
-
@Override
public float[] transformToFloatValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
@@ -247,29 +198,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _floatValuesSV;
}
- @Override
- public Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
- return super.transformToFloatValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initFloatValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _floatValuesSV[i] = (float) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_floatValuesSV, bitmap);
- }
-
@Override
public double[] transformToDoubleValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
@@ -287,29 +215,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _doubleValuesSV;
}
- @Override
- public Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
- return super.transformToDoubleValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initDoubleValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _doubleValuesSV[i] = (double) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_doubleValuesSV, bitmap);
- }
-
@Override
public BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.BIG_DECIMAL) {
@@ -327,29 +232,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _bigDecimalValuesSV;
}
- @Override
- public Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BIG_DECIMAL) {
- return super.transformToBigDecimalValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initBigDecimalValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _bigDecimalValuesSV[i] = (BigDecimal) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_bigDecimalValuesSV, bitmap);
- }
-
@Override
public String[] transformToStringValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
@@ -364,34 +246,11 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
}
Object result = _functionInvoker.invoke(_scalarArguments);
_stringValuesSV[i] =
- _resultType == PinotDataType.STRING ? result.toString() : (String) _resultType.toInternal(result);
+ _resultType == PinotDataType.STRING ? (String) result : (String) _resultType.toInternal(result);
}
return _stringValuesSV;
}
- @Override
- public Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
- return super.transformToStringValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initStringValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _stringValuesSV[i] = (String) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_stringValuesSV, bitmap);
- }
-
@Override
public byte[][] transformToBytesValuesSV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) {
@@ -409,29 +268,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _bytesValuesSV;
}
- @Override
- public Pair<byte[][], RoaringBitmap> transformToBytesValuesSVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) {
- return super.transformToBytesValuesSVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initBytesValuesSV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _bytesValuesSV[i] = (byte[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_bytesValuesSV, bitmap);
- }
-
@Override
public int[][] transformToIntValuesMV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
@@ -449,29 +285,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _intValuesMV;
}
- @Override
- public Pair<int[][], RoaringBitmap> transformToIntValuesMVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
- return super.transformToIntValuesMVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initIntValuesMV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _intValuesMV[i] = (int[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_intValuesMV, bitmap);
- }
-
@Override
public long[][] transformToLongValuesMV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
@@ -489,29 +302,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _longValuesMV;
}
- @Override
- public Pair<long[][], RoaringBitmap> transformToLongValuesMVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
- return super.transformToLongValuesMVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initLongValuesMV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _longValuesMV[i] = (long[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_longValuesMV, bitmap);
- }
-
@Override
public float[][] transformToFloatValuesMV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
@@ -529,29 +319,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _floatValuesMV;
}
- @Override
- public Pair<float[][], RoaringBitmap> transformToFloatValuesMVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
- return super.transformToFloatValuesMVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initFloatValuesMV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _floatValuesMV[i] = (float[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_floatValuesMV, bitmap);
- }
-
@Override
public double[][] transformToDoubleValuesMV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
@@ -569,29 +336,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _doubleValuesMV;
}
- @Override
- public Pair<double[][], RoaringBitmap> transformToDoubleValuesMVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
- return super.transformToDoubleValuesMVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initDoubleValuesMV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _doubleValuesMV[i] = (double[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_doubleValuesMV, bitmap);
- }
-
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
@@ -609,49 +353,6 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
return _stringValuesMV;
}
- @Override
- public Pair<String[][], RoaringBitmap> transformToStringValuesMVWithNull(ValueBlock valueBlock) {
- if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
- return super.transformToStringValuesMVWithNull(valueBlock);
- }
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- initStringValuesMV(length);
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result != null) {
- _stringValuesMV[i] = (String[]) _resultType.toInternal(result);
- } else {
- bitmap.add(i);
- }
- }
- return ImmutablePair.of(_stringValuesMV, bitmap);
- }
-
- @Override
- public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
- int length = valueBlock.getNumDocs();
- RoaringBitmap bitmap = new RoaringBitmap();
- getNonLiteralValuesWithNull(valueBlock);
- for (int i = 0; i < length; i++) {
- for (int j = 0; j < _numNonLiteralArguments; j++) {
- _scalarArguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
- }
- Object result = _functionInvoker.invoke(_scalarArguments);
- if (result == null) {
- bitmap.add(i);
- }
- }
- if (bitmap != null && bitmap.isEmpty()) {
- return null;
- }
- return bitmap;
- }
-
/**
* Helper method to fetch values for the non-literal transform functions based on the parameter types.
*/
@@ -722,111 +423,4 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
}
}
}
-
- /**
- * Helper method to fetch values with null for the non-literal transform functions based on the parameter types.
- */
- private void getNonLiteralValuesWithNull(ValueBlock valueBlock) {
- PinotDataType[] parameterTypes = _functionInvoker.getParameterTypes();
- for (int i = 0; i < _numNonLiteralArguments; i++) {
- PinotDataType parameterType = parameterTypes[_nonLiteralIndices[i]];
- TransformFunction transformFunction = _nonLiteralFunctions[i];
- RoaringBitmap bitmap = null;
- switch (parameterType) {
- case INTEGER:
- Pair<int[], RoaringBitmap> intResult = transformFunction.transformToIntValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = ArrayUtils.toObject(intResult.getLeft());
- bitmap = intResult.getRight();
- break;
- case LONG:
- Pair<long[], RoaringBitmap> longResult = transformFunction.transformToLongValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = ArrayUtils.toObject(longResult.getLeft());
- bitmap = longResult.getRight();
- break;
- case FLOAT:
- Pair<float[], RoaringBitmap> floatResult = transformFunction.transformToFloatValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = ArrayUtils.toObject(floatResult.getLeft());
- bitmap = floatResult.getRight();
- break;
- case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResult = transformFunction.transformToDoubleValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = ArrayUtils.toObject(doubleResult.getLeft());
- bitmap = doubleResult.getRight();
- break;
- case BIG_DECIMAL:
- Pair<BigDecimal[], RoaringBitmap> bigDecimalResult =
- transformFunction.transformToBigDecimalValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = bigDecimalResult.getLeft();
- bitmap = bigDecimalResult.getRight();
- break;
- case BOOLEAN: {
- Pair<int[], RoaringBitmap> boolResult = transformFunction.transformToIntValuesSVWithNull(valueBlock);
- int numValues = boolResult.getLeft().length;
- Boolean[] booleanValues = new Boolean[numValues];
- for (int j = 0; j < numValues; j++) {
- booleanValues[j] = boolResult.getLeft()[j] == 1;
- }
- _nonLiteralValues[i] = booleanValues;
- bitmap = boolResult.getRight();
- break;
- }
- case TIMESTAMP: {
- Pair<long[], RoaringBitmap> timeResult = transformFunction.transformToLongValuesSVWithNull(valueBlock);
- int numValues = timeResult.getLeft().length;
- Timestamp[] timestampValues = new Timestamp[numValues];
- for (int j = 0; j < numValues; j++) {
- timestampValues[j] = new Timestamp(timeResult.getLeft()[j]);
- }
- _nonLiteralValues[i] = timestampValues;
- bitmap = timeResult.getRight();
- break;
- }
- case STRING:
- Pair<String[], RoaringBitmap> stringResult = transformFunction.transformToStringValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = stringResult.getLeft();
- bitmap = stringResult.getRight();
- break;
- case BYTES:
- Pair<byte[][], RoaringBitmap> byteResult = transformFunction.transformToBytesValuesSVWithNull(valueBlock);
- _nonLiteralValues[i] = byteResult.getLeft();
- bitmap = byteResult.getRight();
- break;
- case PRIMITIVE_INT_ARRAY:
- Pair<int[][], RoaringBitmap> intMVResult = transformFunction.transformToIntValuesMVWithNull(valueBlock);
- _nonLiteralValues[i] = intMVResult.getLeft();
- bitmap = intMVResult.getRight();
- break;
- case PRIMITIVE_LONG_ARRAY:
- Pair<long[][], RoaringBitmap> longMVResult = transformFunction.transformToLongValuesMVWithNull(valueBlock);
- _nonLiteralValues[i] = longMVResult.getLeft();
- bitmap = longMVResult.getRight();
- break;
- case PRIMITIVE_FLOAT_ARRAY:
- Pair<float[][], RoaringBitmap> floatMVResult = transformFunction.transformToFloatValuesMVWithNull(valueBlock);
- _nonLiteralValues[i] = floatMVResult.getLeft();
- bitmap = floatMVResult.getRight();
- break;
- case PRIMITIVE_DOUBLE_ARRAY:
- Pair<double[][], RoaringBitmap> doubleMVResult =
- transformFunction.transformToDoubleValuesMVWithNull(valueBlock);
- _nonLiteralValues[i] = doubleMVResult.getLeft();
- bitmap = doubleMVResult.getRight();
- break;
- case STRING_ARRAY:
- Pair<String[][], RoaringBitmap> stringMVResult =
- transformFunction.transformToStringValuesMVWithNull(valueBlock);
- _nonLiteralValues[i] = stringMVResult.getLeft();
- bitmap = stringMVResult.getRight();
- break;
- default:
- throw new IllegalStateException("Unsupported parameter type: " + parameterType);
- }
- if (bitmap != null) {
- int finalI = i;
- bitmap.forEach((IntConsumer) (j) -> {
- _nonLiteralValues[finalI][j] = null;
- });
- }
- }
- }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
index 584efef5fd..83b63931fa 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SelectTupleElementTransformFunction.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.transform.function;
+import java.math.BigDecimal;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.List;
@@ -29,7 +30,8 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.roaringbitmap.RoaringBitmap;
-public abstract class SelectTupleElementTransformFunction extends BaseTransformFunction {
+public abstract class SelectTupleElementTransformFunction
+ extends ComputeDifferentlyWhenNullHandlingEnabledTransformFunction {
private static final EnumSet<FieldSpec.DataType> SUPPORTED_DATATYPES = EnumSet.of(FieldSpec.DataType.INT,
FieldSpec.DataType.LONG, FieldSpec.DataType.FLOAT, FieldSpec.DataType.DOUBLE, FieldSpec.DataType.BIG_DECIMAL,
@@ -48,7 +50,9 @@ public abstract class SelectTupleElementTransformFunction extends BaseTransformF
}
@Override
- public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap) {
+ public void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap,
+ boolean nullHandlingEnabled) {
+ super.init(arguments, columnContextMap, nullHandlingEnabled);
if (arguments.isEmpty()) {
throw new IllegalArgumentException(_name + " takes at least one argument");
}
@@ -131,4 +135,286 @@ public abstract class SelectTupleElementTransformFunction extends BaseTransformF
combinations.put(FieldSpec.DataType.STRING, EnumSet.of(FieldSpec.DataType.STRING));
return combinations;
}
+
+ @Override
+ protected int[] transformToIntValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initIntValuesSV(numDocs);
+ int[] values = _arguments.get(0).transformToIntValuesSV(valueBlock);
+ System.arraycopy(values, 0, _intValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToIntValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _intValuesSV[j] = binaryFunction(_intValuesSV[j], values[j]);
+ }
+ }
+ return _intValuesSV;
+ }
+
+ @Override
+ protected int[] transformToIntValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initIntValuesSV(numDocs);
+ int[] curValues = _arguments.get(0).transformToIntValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _intValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToIntValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _intValuesSV[j] = curValues[j];
+ } else {
+ _intValuesSV[j] = binaryFunction(_intValuesSV[j], curValues[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _intValuesSV;
+ }
+
+ abstract protected int binaryFunction(int a, int b);
+
+ @Override
+ protected long[] transformToLongValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initLongValuesSV(numDocs);
+ long[] values = _arguments.get(0).transformToLongValuesSV(valueBlock);
+ System.arraycopy(values, 0, _longValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToLongValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _longValuesSV[j] = binaryFunction(_longValuesSV[j], values[j]);
+ }
+ }
+ return _longValuesSV;
+ }
+
+ @Override
+ protected long[] transformToLongValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initLongValuesSV(numDocs);
+ long[] curValues = _arguments.get(0).transformToLongValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _longValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToLongValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _longValuesSV[j] = curValues[j];
+ } else {
+ _longValuesSV[j] = binaryFunction(_longValuesSV[j], curValues[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _longValuesSV;
+ }
+
+ abstract protected long binaryFunction(long a, long b);
+
+ @Override
+ protected float[] transformToFloatValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initFloatValuesSV(numDocs);
+ float[] values = _arguments.get(0).transformToFloatValuesSV(valueBlock);
+ System.arraycopy(values, 0, _floatValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToFloatValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _floatValuesSV[j] = binaryFunction(_floatValuesSV[j], values[j]);
+ }
+ }
+ return _floatValuesSV;
+ }
+
+ @Override
+ protected float[] transformToFloatValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initFloatValuesSV(numDocs);
+ float[] curValues = _arguments.get(0).transformToFloatValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _floatValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToFloatValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _floatValuesSV[j] = curValues[j];
+ } else {
+ _floatValuesSV[j] = binaryFunction(_floatValuesSV[j], curValues[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _floatValuesSV;
+ }
+
+ abstract protected float binaryFunction(float a, float b);
+
+ @Override
+ protected double[] transformToDoubleValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initDoubleValuesSV(numDocs);
+ double[] values = _arguments.get(0).transformToDoubleValuesSV(valueBlock);
+ System.arraycopy(values, 0, _doubleValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToDoubleValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _doubleValuesSV[j] = binaryFunction(_doubleValuesSV[j], values[j]);
+ }
+ }
+ return _doubleValuesSV;
+ }
+
+ @Override
+ protected double[] transformToDoubleValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initDoubleValuesSV(numDocs);
+ double[] curValues = _arguments.get(0).transformToDoubleValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _doubleValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToDoubleValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _doubleValuesSV[j] = curValues[j];
+ } else {
+ _doubleValuesSV[j] = binaryFunction(_doubleValuesSV[j], curValues[j]);
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _doubleValuesSV;
+ }
+
+ abstract protected double binaryFunction(double a, double b);
+
+ @Override
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initBigDecimalValuesSV(numDocs);
+ BigDecimal[] values = _arguments.get(0).transformToBigDecimalValuesSV(valueBlock);
+ System.arraycopy(values, 0, _bigDecimalValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToBigDecimalValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _bigDecimalValuesSV[j] = binaryFunction(_bigDecimalValuesSV[j], values[j]);
+ }
+ }
+ return _bigDecimalValuesSV;
+ }
+
+ @Override
+ protected BigDecimal[] transformToBigDecimalValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initBigDecimalValuesSV(numDocs);
+ BigDecimal[] curValues = _arguments.get(0).transformToBigDecimalValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _bigDecimalValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToBigDecimalValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _bigDecimalValuesSV[j] = curValues[j];
+ } else {
+ _bigDecimalValuesSV[j] = binaryFunction(_bigDecimalValuesSV[j], (curValues[j]));
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _bigDecimalValuesSV;
+ }
+
+ abstract protected BigDecimal binaryFunction(BigDecimal a, BigDecimal b);
+
+ @Override
+ protected String[] transformToStringValuesSVUsingValue(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initStringValuesSV(numDocs);
+ String[] values = _arguments.get(0).transformToStringValuesSV(valueBlock);
+ System.arraycopy(values, 0, _stringValuesSV, 0, numDocs);
+ for (int i = 1; i < _arguments.size(); i++) {
+ values = _arguments.get(i).transformToStringValuesSV(valueBlock);
+ for (int j = 0; j < numDocs & j < values.length; j++) {
+ _stringValuesSV[j] = binaryFunction(_stringValuesSV[j], (values[j]));
+ }
+ }
+ return _stringValuesSV;
+ }
+
+ @Override
+ protected String[] transformToStringValuesSVUsingValueAndNull(ValueBlock valueBlock) {
+ int numDocs = valueBlock.getNumDocs();
+ initStringValuesSV(numDocs);
+ String[] curValues = _arguments.get(0).transformToStringValuesSV(valueBlock);
+ System.arraycopy(curValues, 0, _stringValuesSV, 0, numDocs);
+ RoaringBitmap nullBitmap = _arguments.get(0).getNullBitmap(valueBlock);
+ for (int i = 1; i < _arguments.size(); i++) {
+ curValues = _arguments.get(i).transformToStringValuesSV(valueBlock);
+ RoaringBitmap curNull = _arguments.get(i).getNullBitmap(valueBlock);
+ for (int j = 0; j < numDocs & j < curValues.length; j++) {
+ // If current value is not null, we process the data.
+ if (curNull == null || !curNull.contains(j)) {
+ // If existing maximum value is null, we set the value directly.
+ if (nullBitmap != null && nullBitmap.contains(j)) {
+ _stringValuesSV[j] = curValues[j];
+ } else {
+ _stringValuesSV[j] = binaryFunction(_stringValuesSV[j], (curValues[j]));
+ }
+ }
+ }
+ if (nullBitmap != null && curNull != null) {
+ nullBitmap.and(curNull);
+ } else {
+ nullBitmap = null;
+ }
+ }
+ return _stringValuesSV;
+ }
+
+ abstract protected String binaryFunction(String a, String b);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
index 7f3e59009d..95ee40f727 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
@@ -22,8 +22,6 @@ import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
@@ -52,6 +50,18 @@ public interface TransformFunction {
*/
void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap);
+ /**
+ * Initializes the transform function.
+ *
+ * @param arguments Arguments for the transform function
+ * @param columnContextMap Map from column name to context
+ * @param nullHandlingEnabled Whether this transform function handles {@code null}
+ */
+ default void init(List<TransformFunction> arguments, Map<String, ColumnContext> columnContextMap,
+ boolean nullHandlingEnabled) {
+ init(arguments, columnContextMap);
+ }
+
/**
* Returns the metadata for the result of the transform function.
*
@@ -74,25 +84,11 @@ public interface TransformFunction {
*/
int[] transformToDictIdsSV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to single-valued dictionary ids with null bit vector.
- */
- default Pair<int[], RoaringBitmap> transformToDictIdsSVWithNull(ValueBlock block) {
- return ImmutablePair.of(transformToDictIdsSV(block), getNullBitmap(block));
- }
-
/**
* Transforms the data from the given value block to multi-valued dictionary ids.
*/
int[][] transformToDictIdsMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to multi-valued dictionary ids with null bit vector.
- */
- default Pair<int[][], RoaringBitmap> transformToDictIdsMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToDictIdsMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* SINGLE-VALUED APIs
*/
@@ -102,88 +98,36 @@ public interface TransformFunction {
*/
int[] transformToIntValuesSV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to single-valued int values with null bit vector.
- */
- default Pair<int[], RoaringBitmap> transformToIntValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToIntValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued long values.
*/
long[] transformToLongValuesSV(ValueBlock valueBlock);
-
- /**
- * Transforms the data from the given value block to single-valued long values with null bit vector.
- */
- default Pair<long[], RoaringBitmap> transformToLongValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToLongValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued float values.
*/
float[] transformToFloatValuesSV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to single-valued float values with null bit vector.
- */
- default Pair<float[], RoaringBitmap> transformToFloatValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToFloatValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued double values.
*/
double[] transformToDoubleValuesSV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to single-valued double values with null bit vector.
- */
- default Pair<double[], RoaringBitmap> transformToDoubleValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToDoubleValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued BigDecimal values.
*/
BigDecimal[] transformToBigDecimalValuesSV(ValueBlock valueBlock);
-
- /**
- * Transforms the data from the given projection block to single-valued BigDecimal values and null bit vector.
- */
- default Pair<BigDecimal[], RoaringBitmap> transformToBigDecimalValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToBigDecimalValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued string values.
*/
String[] transformToStringValuesSV(ValueBlock valueBlock);
-
- /**
- * Transforms the data from the given projection block to single-valued string values and null bit vector.
- */
- default Pair<String[], RoaringBitmap> transformToStringValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToStringValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to single-valued bytes values.
*/
byte[][] transformToBytesValuesSV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given projection block to single-valued bytes values and null bit vector.
- */
- default Pair<byte[][], RoaringBitmap> transformToBytesValuesSVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToBytesValuesSV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* MULTI-VALUED APIs
*/
@@ -193,74 +137,31 @@ public interface TransformFunction {
*/
int[][] transformToIntValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to multi-valued double values and null bit vector.
- */
- default Pair<int[][], RoaringBitmap> transformToIntValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToIntValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to multi-valued long values.
*/
long[][] transformToLongValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to multi-valued double values and null bit vector.
- */
- default Pair<long[][], RoaringBitmap> transformToLongValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToLongValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to multi-valued float values.
*/
float[][] transformToFloatValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given value block to multi-valued double values and null bit vector.
- */
- default Pair<float[][], RoaringBitmap> transformToFloatValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToFloatValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to multi-valued double values.
*/
double[][] transformToDoubleValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given projection block to multi-valued double values and null bit vector.
- */
- default Pair<double[][], RoaringBitmap> transformToDoubleValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToDoubleValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Transforms the data from the given value block to multi-valued string values.
*/
String[][] transformToStringValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given projection block to multi-valued string values and null bit vector.
- */
- default Pair<String[][], RoaringBitmap> transformToStringValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToStringValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
-
/**
* Transforms the data from the given value block to multi-valued bytes values.
*/
byte[][][] transformToBytesValuesMV(ValueBlock valueBlock);
- /**
- * Transforms the data from the given projection block to multi-valued bytes values and null bit vector.
- */
- default Pair<byte[][][], RoaringBitmap> transformToBytesValuesMVWithNull(ValueBlock valueBlock) {
- return ImmutablePair.of(transformToBytesValuesMV(valueBlock), getNullBitmap(valueBlock));
- }
-
/**
* Gets the null rows for transformation result. Should be called when only null information is needed for
* transformation.
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
index 0ac3fafaa3..c4364425e1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.function.FunctionInfo;
import org.apache.pinot.common.function.FunctionRegistry;
@@ -72,8 +71,10 @@ import org.apache.pinot.core.operator.transform.function.TrigonometricTransformF
import org.apache.pinot.core.operator.transform.function.TrigonometricTransformFunctions.TanTransformFunction;
import org.apache.pinot.core.operator.transform.function.TrigonometricTransformFunctions.TanhTransformFunction;
import org.apache.pinot.core.query.request.context.QueryContext;
+import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -254,11 +255,11 @@ public class TransformFunctionFactory {
*
* @param expression Transform expression
* @param columnContextMap Map from column name to context
- * @param queryContext Query context if available
+ * @param queryContext Query context
* @return Transform function
*/
public static TransformFunction get(ExpressionContext expression, Map<String, ColumnContext> columnContextMap,
- @Nullable QueryContext queryContext) {
+ QueryContext queryContext) {
switch (expression.getType()) {
case FUNCTION:
FunctionContext function = expression.getFunction();
@@ -294,7 +295,7 @@ public class TransformFunctionFactory {
transformFunctionArguments.add(TransformFunctionFactory.get(argument, columnContextMap, queryContext));
}
try {
- transformFunction.init(transformFunctionArguments, columnContextMap);
+ transformFunction.init(transformFunctionArguments, columnContextMap, queryContext.isNullHandlingEnabled());
} catch (Exception e) {
throw new BadQueryRequestException("Caught exception while initializing transform function: " + functionName,
e);
@@ -304,8 +305,7 @@ public class TransformFunctionFactory {
String columnName = expression.getIdentifier();
return new IdentifierTransformFunction(columnName, columnContextMap.get(columnName));
case LITERAL:
- return queryContext == null ? new LiteralTransformFunction(expression.getLiteral())
- : queryContext.getOrComputeSharedValue(LiteralTransformFunction.class, expression.getLiteral(),
+ return queryContext.getOrComputeSharedValue(LiteralTransformFunction.class, expression.getLiteral(),
LiteralTransformFunction::new);
default:
throw new IllegalStateException();
@@ -316,7 +316,19 @@ public class TransformFunctionFactory {
public static TransformFunction get(ExpressionContext expression, Map<String, DataSource> dataSourceMap) {
Map<String, ColumnContext> columnContextMap = new HashMap<>(HashUtil.getHashMapCapacity(dataSourceMap.size()));
dataSourceMap.forEach((k, v) -> columnContextMap.put(k, ColumnContext.fromDataSource(v)));
- return get(expression, columnContextMap, null);
+ QueryContext dummy = QueryContextConverterUtils.getQueryContext(
+ CalciteSqlParser.compileToPinotQuery("SELECT * from testTable;"));
+ return get(expression, columnContextMap, dummy);
+ }
+
+ @VisibleForTesting
+ public static TransformFunction getNullHandlingEnabled(ExpressionContext expression,
+ Map<String, DataSource> dataSourceMap) {
+ Map<String, ColumnContext> columnContextMap = new HashMap<>(HashUtil.getHashMapCapacity(dataSourceMap.size()));
+ dataSourceMap.forEach((k, v) -> columnContextMap.put(k, ColumnContext.fromDataSource(v)));
+ QueryContext dummy = QueryContextConverterUtils.getQueryContext(
+ CalciteSqlParser.compileToPinotQuery("SET enableNullHandling = true; SELECT * from testTable;"));
+ return get(expression, columnContextMap, dummy);
}
/**
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java
index 86c7e28907..ea9dd79fe5 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
import org.apache.pinot.spi.data.FieldSpec;
@@ -89,84 +88,54 @@ public abstract class ArrayBaseTransformFunctionTest extends BaseTransformFuncti
Assert.assertTrue(transformFunction.getResultMetadata().isSingleValue());
Assert.assertFalse(transformFunction.getResultMetadata().hasDictionary());
+ RoaringBitmap expectedNulls = new RoaringBitmap();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (isNullRow(i)) {
+ expectedNulls.add(i);
+ }
+ }
+ testNullBitmap(transformFunction, expectedNulls);
+
switch (getResultDataType(FieldSpec.DataType.INT)) {
case INT:
- Pair<int[], RoaringBitmap> intResults = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- int[] intValues = intResults.getLeft();
- RoaringBitmap nullBitmap = intResults.getRight();
- RoaringBitmap expectedNulls = new RoaringBitmap();
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
Assert.assertEquals(intValues[i], getExpectResult(_intMVValues[i]));
- } else {
- expectedNulls.add(i);
}
}
- Assert.assertEquals(nullBitmap, expectedNulls);
- Assert.assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNulls);
break;
case LONG:
- Pair<long[], RoaringBitmap> longResults = transformFunction.transformToLongValuesSVWithNull(_projectionBlock);
- long[] longValues = longResults.getLeft();
- nullBitmap = longResults.getRight();
- expectedNulls = new RoaringBitmap();
+ long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
Assert.assertEquals(longValues[i], getExpectResult(_intMVValues[i]));
- } else {
- expectedNulls.add(i);
}
}
- Assert.assertEquals(nullBitmap, expectedNulls);
- Assert.assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNulls);
break;
case FLOAT:
- Pair<float[], RoaringBitmap> floatResults =
- transformFunction.transformToFloatValuesSVWithNull(_projectionBlock);
- float[] floatValues = floatResults.getLeft();
- nullBitmap = floatResults.getRight();
- expectedNulls = new RoaringBitmap();
+ float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
Assert.assertEquals(floatValues[i], getExpectResult(_intMVValues[i]));
- } else {
- expectedNulls.add(i);
}
}
- Assert.assertEquals(nullBitmap, expectedNulls);
- Assert.assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNulls);
break;
case DOUBLE:
- Pair<double[], RoaringBitmap> doubleResults =
- transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
- double[] doubleValues = doubleResults.getLeft();
- nullBitmap = doubleResults.getRight();
- expectedNulls = new RoaringBitmap();
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
Assert.assertEquals(doubleValues[i], getExpectResult(_intMVValues[i]));
- } else {
- expectedNulls.add(i);
}
}
- Assert.assertEquals(nullBitmap, expectedNulls);
- Assert.assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNulls);
break;
case STRING:
- Pair<String[], RoaringBitmap> stringResults =
- transformFunction.transformToStringValuesSVWithNull(_projectionBlock);
- String[] stringValues = stringResults.getLeft();
- nullBitmap = stringResults.getRight();
- expectedNulls = new RoaringBitmap();
+ String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
Assert.assertEquals(stringValues[i], getExpectResult(_intMVValues[i]));
- } else {
- expectedNulls.add(i);
}
}
- Assert.assertEquals(nullBitmap, expectedNulls);
- Assert.assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNulls);
break;
default:
break;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
index 9d79f77cec..6e3c139aee 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
@@ -34,7 +34,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.core.operator.DocIdSetOperator;
import org.apache.pinot.core.operator.ProjectionOperator;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
@@ -265,7 +264,7 @@ public abstract class BaseTransformFunctionTest {
return i % 2 != 0;
}
- private void testNullBitmap(TransformFunction transformFunction, RoaringBitmap expectedNull) {
+ protected void testNullBitmap(TransformFunction transformFunction, RoaringBitmap expectedNull) {
RoaringBitmap nullBitmap = transformFunction.getNullBitmap(_projectionBlock);
assertEquals(nullBitmap, expectedNull);
}
@@ -290,30 +289,24 @@ public abstract class BaseTransformFunctionTest {
protected void testTransformFunctionWithNull(TransformFunction transformFunction, int[] expectedValues,
RoaringBitmap expectedNull) {
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- Pair<long[], RoaringBitmap> longValues = transformFunction.transformToLongValuesSVWithNull(_projectionBlock);
- Pair<float[], RoaringBitmap> floatValues = transformFunction.transformToFloatValuesSVWithNull(_projectionBlock);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
- Pair<BigDecimal[], RoaringBitmap> bigDecimalValues =
- transformFunction.transformToBigDecimalValuesSVWithNull(_projectionBlock);
- Pair<String[], RoaringBitmap> stringValues = transformFunction.transformToStringValuesSVWithNull(_projectionBlock);
- assertEquals(intValues.getRight(), expectedNull);
- assertEquals(longValues.getRight(), expectedNull);
- assertEquals(floatValues.getRight(), expectedNull);
- assertEquals(doubleValues.getRight(), expectedNull);
- assertEquals(bigDecimalValues.getRight(), expectedNull);
- assertEquals(stringValues.getRight(), expectedNull);
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
+ long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock);
+ float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
+ BigDecimal[] bigDecimalValues =
+ transformFunction.transformToBigDecimalValuesSV(_projectionBlock);
+ String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (expectedNull.contains(i)) {
continue;
}
// only compare the rows that are not null.
- assertEquals(intValues.getLeft()[i], expectedValues[i]);
- assertEquals(longValues.getLeft()[i], expectedValues[i]);
- assertEquals(floatValues.getLeft()[i], (float) expectedValues[i]);
- assertEquals(doubleValues.getLeft()[i], (double) expectedValues[i]);
- assertEquals(bigDecimalValues.getLeft()[i].intValue(), expectedValues[i]);
- assertEquals(stringValues.getLeft()[i], Integer.toString(expectedValues[i]));
+ assertEquals(intValues[i], expectedValues[i]);
+ assertEquals(longValues[i], expectedValues[i]);
+ assertEquals(floatValues[i], (float) expectedValues[i]);
+ assertEquals(doubleValues[i], (double) expectedValues[i]);
+ assertEquals(bigDecimalValues[i].intValue(), expectedValues[i]);
+ assertEquals(stringValues[i], Integer.toString(expectedValues[i]));
}
testNullBitmap(transformFunction, expectedNull);
}
@@ -338,30 +331,24 @@ public abstract class BaseTransformFunctionTest {
protected void testTransformFunctionWithNull(TransformFunction transformFunction, long[] expectedValues,
RoaringBitmap expectedNull) {
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- Pair<long[], RoaringBitmap> longValues = transformFunction.transformToLongValuesSVWithNull(_projectionBlock);
- Pair<float[], RoaringBitmap> floatValues = transformFunction.transformToFloatValuesSVWithNull(_projectionBlock);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
- Pair<BigDecimal[], RoaringBitmap> bigDecimalValues =
- transformFunction.transformToBigDecimalValuesSVWithNull(_projectionBlock);
- Pair<String[], RoaringBitmap> stringValues = transformFunction.transformToStringValuesSVWithNull(_projectionBlock);
- assertEquals(intValues.getRight(), expectedNull);
- assertEquals(longValues.getRight(), expectedNull);
- assertEquals(floatValues.getRight(), expectedNull);
- assertEquals(doubleValues.getRight(), expectedNull);
- assertEquals(bigDecimalValues.getRight(), expectedNull);
- assertEquals(stringValues.getRight(), expectedNull);
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
+ long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock);
+ float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
+ BigDecimal[] bigDecimalValues =
+ transformFunction.transformToBigDecimalValuesSV(_projectionBlock);
+ String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (expectedNull.contains(i)) {
continue;
}
// only compare the rows that are not null.
- assertEquals(intValues.getLeft()[i], (int) expectedValues[i]);
- assertEquals(longValues.getLeft()[i], expectedValues[i]);
- assertEquals(floatValues.getLeft()[i], (float) expectedValues[i]);
- assertEquals(doubleValues.getLeft()[i], (double) expectedValues[i]);
- assertEquals(bigDecimalValues.getLeft()[i].longValue(), expectedValues[i]);
- assertEquals(stringValues.getLeft()[i], Long.toString(expectedValues[i]));
+ assertEquals(intValues[i], (int) expectedValues[i]);
+ assertEquals(longValues[i], expectedValues[i]);
+ assertEquals(floatValues[i], (float) expectedValues[i]);
+ assertEquals(doubleValues[i], (double) expectedValues[i]);
+ assertEquals(bigDecimalValues[i].longValue(), expectedValues[i]);
+ assertEquals(stringValues[i], Long.toString(expectedValues[i]));
}
testNullBitmap(transformFunction, expectedNull);
}
@@ -413,40 +400,32 @@ public abstract class BaseTransformFunctionTest {
protected void testTransformFunctionWithNull(TransformFunction transformFunction, double[] expectedValues,
RoaringBitmap expectedNull) {
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- Pair<long[], RoaringBitmap> longValues = transformFunction.transformToLongValuesSVWithNull(_projectionBlock);
- Pair<float[], RoaringBitmap> floatValues = transformFunction.transformToFloatValuesSVWithNull(_projectionBlock);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
- Pair<BigDecimal[], RoaringBitmap> bigDecimalValues = null;
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
+ long[]longValues = transformFunction.transformToLongValuesSV(_projectionBlock);
+ float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock);
+ double[]doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
+ BigDecimal[] bigDecimalValues = null;
try {
// 1- Some transform functions cannot work with BigDecimal (e.g. exp, ln, and sqrt).
// 2- NumberFormatException is thrown when converting double.NaN, Double.POSITIVE_INFINITY,
// or Double.NEGATIVE_INFINITY.
- bigDecimalValues = transformFunction.transformToBigDecimalValuesSVWithNull(_projectionBlock);
+ bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock);
} catch (UnsupportedOperationException | NumberFormatException ignored) {
}
- Pair<String[], RoaringBitmap> stringValues = transformFunction.transformToStringValuesSVWithNull(_projectionBlock);
- assertEquals(intValues.getRight(), expectedNull);
- assertEquals(longValues.getRight(), expectedNull);
- assertEquals(floatValues.getRight(), expectedNull);
- assertEquals(doubleValues.getRight(), expectedNull);
- if (bigDecimalValues != null) {
- assertEquals(bigDecimalValues.getRight(), expectedNull);
- }
- assertEquals(stringValues.getRight(), expectedNull);
+ String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
// only compare the results for non-null rows.
if (expectedNull.contains(i)) {
continue;
}
- assertEquals(intValues.getLeft()[i], (int) expectedValues[i]);
- assertEquals(longValues.getLeft()[i], (long) expectedValues[i]);
- assertEquals(floatValues.getLeft()[i], (float) expectedValues[i]);
- assertEquals(doubleValues.getLeft()[i], expectedValues[i]);
+ assertEquals(intValues[i], (int) expectedValues[i]);
+ assertEquals(longValues[i], (long) expectedValues[i]);
+ assertEquals(floatValues[i], (float) expectedValues[i]);
+ assertEquals(doubleValues[i], expectedValues[i]);
if (bigDecimalValues != null) {
- assertEquals(bigDecimalValues.getLeft()[i].doubleValue(), expectedValues[i]);
+ assertEquals(bigDecimalValues[i].doubleValue(), expectedValues[i]);
}
- assertEquals(stringValues.getLeft()[i], Double.toString(expectedValues[i]));
+ assertEquals(stringValues[i], Double.toString(expectedValues[i]));
}
testNullBitmap(transformFunction, expectedNull);
}
@@ -490,27 +469,22 @@ public abstract class BaseTransformFunctionTest {
protected void testTransformFunctionWithNull(TransformFunction transformFunction, boolean[] expectedValues,
RoaringBitmap expectedNulls) {
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- Pair<long[], RoaringBitmap> longValues = transformFunction.transformToLongValuesSVWithNull(_projectionBlock);
- Pair<float[], RoaringBitmap> floatValues = transformFunction.transformToFloatValuesSVWithNull(_projectionBlock);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
- Pair<BigDecimal[], RoaringBitmap> bigDecimalValues =
- transformFunction.transformToBigDecimalValuesSVWithNull(_projectionBlock);
- assertEquals(intValues.getRight(), expectedNulls);
- assertEquals(longValues.getRight(), expectedNulls);
- assertEquals(floatValues.getRight(), expectedNulls);
- assertEquals(doubleValues.getRight(), expectedNulls);
- assertEquals(bigDecimalValues.getRight(), expectedNulls);
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
+ long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock);
+ float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
+ BigDecimal[]bigDecimalValues =
+ transformFunction.transformToBigDecimalValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (expectedNulls.contains(i)) {
continue;
}
- assertEquals(intValues.getLeft()[i] == 1, expectedValues[i]);
- assertEquals(longValues.getLeft()[i] == 1, expectedValues[i]);
- assertEquals(floatValues.getLeft()[i] == 1, expectedValues[i]);
- assertEquals(doubleValues.getLeft()[i] == 1, expectedValues[i]);
- assertEquals(bigDecimalValues.getLeft()[i].intValue() == 1, expectedValues[i]);
+ assertEquals(intValues[i] == 1, expectedValues[i]);
+ assertEquals(longValues[i] == 1, expectedValues[i]);
+ assertEquals(floatValues[i] == 1, expectedValues[i]);
+ assertEquals(doubleValues[i] == 1, expectedValues[i]);
+ assertEquals(bigDecimalValues[i].intValue() == 1, expectedValues[i]);
}
testNullBitmap(transformFunction, expectedNulls);
}
@@ -544,11 +518,14 @@ public abstract class BaseTransformFunctionTest {
protected void testTransformFunctionWithNull(TransformFunction transformFunction, String[] expectedValues,
RoaringBitmap expectedNulls) {
- Pair<String[], RoaringBitmap> stringValues = transformFunction.transformToStringValuesSVWithNull(_projectionBlock);
+ String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
- assertEquals(stringValues.getLeft()[i], expectedValues[i]);
+ if (expectedNulls.contains(i)) {
+ continue;
+ }
+ assertEquals(stringValues[i], expectedValues[i]);
}
- assertEquals(stringValues.getRight(), expectedNulls);
+ testNullBitmap(transformFunction, expectedNulls);
}
protected void testTransformFunction(TransformFunction transformFunction, byte[][] expectedValues) {
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
index 31f786ca3c..cf1c055591 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
@@ -19,7 +19,6 @@
package org.apache.pinot.core.operator.transform.function;
import java.util.function.LongToIntFunction;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.scalar.DateTimeFunctions;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
@@ -99,13 +98,14 @@ public class DateTimeTransformFunctionTest extends BaseTransformFunctionTest {
RequestContextUtils.getExpression(String.format("%s(%s)", function, TIMESTAMP_COLUMN_NULL));
TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
Assert.assertTrue(expectedClass.isInstance(transformFunction));
- Pair<int[], RoaringBitmap> values = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ int[] values = transformFunction.transformToIntValuesSV(_projectionBlock);
+ RoaringBitmap nullBitmap = transformFunction.getNullBitmap(_projectionBlock);
for (int i = 0; i < _projectionBlock.getNumDocs(); i++) {
if (i % 2 == 0) {
- assertEquals(values.getLeft()[i], expected.applyAsInt(_timeValues[i]));
+ assertEquals(values[i], expected.applyAsInt(_timeValues[i]));
} else {
- assertTrue(values.getRight().contains(i));
+ assertTrue(nullBitmap.contains(i));
}
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
index a821d22f98..146e0c3915 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
@@ -19,7 +19,6 @@
package org.apache.pinot.core.operator.transform.function;
import java.util.function.LongToIntFunction;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.scalar.DateTimeFunctions;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
@@ -76,12 +75,13 @@ public class ExtractTransformFunctionTest extends BaseTransformFunctionTest {
TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
Assert.assertTrue(transformFunction instanceof ExtractTransformFunction);
- Pair<int[], RoaringBitmap> valuesSVWithNull = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ int[] values = transformFunction.transformToIntValuesSV(_projectionBlock);
+ RoaringBitmap nullBitmap = transformFunction.getNullBitmap(_projectionBlock);
for (int i = 0; i < _projectionBlock.getNumDocs(); i++) {
if (i % 2 == 0) {
- assertEquals(valuesSVWithNull.getLeft()[i], expected.applyAsInt(_timeValues[i]));
+ assertEquals(values[i], expected.applyAsInt(_timeValues[i]));
} else {
- assertTrue(valuesSVWithNull.getRight().contains(i));
+ assertTrue(nullBitmap.contains(i));
}
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunctionTest.java
index ed2624a5b9..69b838b6db 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunctionTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
@@ -103,8 +102,6 @@ public class IdentifierTransformFunctionTest {
new IdentifierTransformFunction(TEST_COLUMN_NAME, _columnContext);
RoaringBitmap bitmap = identifierTransformFunction.getNullBitmap(_projectionBlock);
Assert.assertEquals(bitmap, NULL_BITMAP);
- Pair<int[], RoaringBitmap> intResult = identifierTransformFunction.transformToIntValuesSVWithNull(_projectionBlock);
- Assert.assertEquals(intResult.getLeft(), INT_VALUES);
- Assert.assertEquals(intResult.getRight(), NULL_BITMAP);
+ Assert.assertEquals(identifierTransformFunction.transformToIntValuesSV(_projectionBlock), INT_VALUES);
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
index dcc0f6ef4f..f33b1138bd 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
@@ -22,7 +22,6 @@ import com.google.common.collect.Sets;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
@@ -331,10 +330,8 @@ public class InTransformFunctionTest extends BaseTransformFunctionTest {
ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
- Pair<int[], RoaringBitmap> intValuesWithNull = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
-
- assertEquals(intValuesWithNull.getLeft()[0], 1);
- assertFalse(intValuesWithNull.getRight().contains(0));
+ assertEquals(transformFunction.transformToIntValuesSV(_projectionBlock)[0], 1);
+ assertFalse(transformFunction.getNullBitmap(_projectionBlock).contains(0));
}
@Test
@@ -344,7 +341,7 @@ public class InTransformFunctionTest extends BaseTransformFunctionTest {
ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
- assertTrue(transformFunction.transformToIntValuesSVWithNull(_projectionBlock).getRight().contains(0));
+ assertTrue(transformFunction.getNullBitmap(_projectionBlock).contains(0));
}
@Test
@@ -367,7 +364,7 @@ public class InTransformFunctionTest extends BaseTransformFunctionTest {
RoaringBitmap expectedNullBitmap = new RoaringBitmap();
expectedNullBitmap.add((long) 0, (long) NUM_ROWS);
- assertEquals(transformFunction.transformToIntValuesSVWithNull(_projectionBlock).getRight(), expectedNullBitmap);
+ testNullBitmap(transformFunction, expectedNullBitmap);
}
@Test
@@ -378,7 +375,7 @@ public class InTransformFunctionTest extends BaseTransformFunctionTest {
RoaringBitmap expectedNullBitmap = new RoaringBitmap();
expectedNullBitmap.add((long) 0, (long) NUM_ROWS);
- assertEquals(transformFunction.transformToIntValuesSVWithNull(_projectionBlock).getRight(), expectedNullBitmap);
+ testNullBitmap(transformFunction, expectedNullBitmap);
}
@Test
@@ -389,6 +386,6 @@ public class InTransformFunctionTest extends BaseTransformFunctionTest {
RoaringBitmap expectedNullBitmap = new RoaringBitmap();
expectedNullBitmap.add((long) 0, (long) NUM_ROWS);
- assertEquals(transformFunction.transformToIntValuesSVWithNull(_projectionBlock).getRight(), expectedNullBitmap);
+ testNullBitmap(transformFunction, expectedNullBitmap);
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunctionTest.java
index abb84b0adc..b1d7d55343 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunctionTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.LiteralContext;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -72,12 +71,11 @@ public class LiteralTransformFunctionTest {
RoaringBitmap expectedBitmap = new RoaringBitmap();
expectedBitmap.add(0L, NUM_DOCS);
Assert.assertEquals(bitmap, expectedBitmap);
- Pair<int[], RoaringBitmap> intResult = nullLiteral.transformToIntValuesSVWithNull(_projectionBlock);
- int[] intValues = intResult.getLeft();
+ int[] intValues = nullLiteral.transformToIntValuesSV(_projectionBlock);
Assert.assertEquals(intValues.length, NUM_DOCS);
for (int i = 0; i < NUM_DOCS; i++) {
Assert.assertEquals(intValues[i], 0);
}
- Assert.assertEquals(intResult.getRight(), expectedBitmap);
+ Assert.assertEquals(nullLiteral.getNullBitmap(_projectionBlock), expectedBitmap);
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
index ffd131d849..237444c6c9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TupleSelectionTransformFunctionsTest.java
@@ -19,7 +19,6 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
@@ -218,61 +217,57 @@ public class TupleSelectionTransformFunctionsTest extends BaseTransformFunctionT
@Test
public void testLeastTransformFunctionNullLiteral() {
- TransformFunction transformFunction =
- testLeastPreconditions(String.format("least(%s, null, %s)", INT_SV_COLUMN, DOUBLE_SV_COLUMN));
+ TransformFunction transformFunction = testLeastPreconditionsNullHandlingEnabled(
+ String.format("least(%s, null, %s)", INT_SV_COLUMN, DOUBLE_SV_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.DOUBLE);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
- assertEquals(doubleValues.getLeft()[i], Math.min(_intSVValues[i], _doubleSVValues[i]));
+ assertEquals(doubleValues[i], Math.min(_intSVValues[i], _doubleSVValues[i]));
}
- assertEquals(doubleValues.getRight(), null);
assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
}
@Test
public void testLeastTransformFunctionNullColumn() {
- TransformFunction transformFunction =
- testLeastPreconditions(String.format("least(%s, null, %s)", INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
+ TransformFunction transformFunction = testLeastPreconditionsNullHandlingEnabled(
+ String.format("least(%s, null, %s)", INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.DOUBLE);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
- assertEquals(doubleValues.getLeft()[i], Math.min(_intSVValues[i], _doubleSVValues[i]));
+ assertEquals(doubleValues[i], Math.min(_intSVValues[i], _doubleSVValues[i]));
} else {
- assertEquals(doubleValues.getLeft()[i], _doubleSVValues[i]);
+ assertEquals(doubleValues[i], _doubleSVValues[i]);
}
}
- assertEquals(doubleValues.getRight(), null);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ testNullBitmap(transformFunction, null);
}
@Test
public void testLeastTransformFunctionAllNulls() {
- TransformFunction transformFunction = testLeastPreconditions(String.format("least(null, null, null)"));
+ TransformFunction transformFunction =
+ testLeastPreconditionsNullHandlingEnabled(String.format("least(null, null, null)"));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.UNKNOWN);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
RoaringBitmap expectedNull = new RoaringBitmap();
expectedNull.add(0L, NUM_ROWS);
- assertEquals(doubleValues.getRight(), expectedNull);
assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNull);
}
@Test
public void testLeastTransformFunctionPartialAllNulls() {
- TransformFunction transformFunction = testLeastPreconditions(
+ TransformFunction transformFunction = testLeastPreconditionsNullHandlingEnabled(
String.format("least(%s, %s, %s)", INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.INT);
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
RoaringBitmap expectedNull = new RoaringBitmap();
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
- assertEquals(intValues.getLeft()[i], _intSVValues[i]);
+ assertEquals(intValues[i], _intSVValues[i]);
} else {
expectedNull.add(i);
}
}
- assertEquals(intValues.getRight(), expectedNull);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNull);
+ testNullBitmap(transformFunction, expectedNull);
}
@Test(dataProvider = "rejectedParameters", expectedExceptions = BadQueryRequestException.class)
@@ -366,61 +361,58 @@ public class TupleSelectionTransformFunctionsTest extends BaseTransformFunctionT
@Test
public void testGreatestTransformFunctionNullLiteral() {
- TransformFunction transformFunction =
- testGreatestPreconditions(String.format("greatest(%s, null, %s)", INT_SV_COLUMN, DOUBLE_SV_COLUMN));
+ TransformFunction transformFunction = testGreatestPreconditionsNullHandlingEnabled(
+ String.format("greatest(%s, null, %s)", INT_SV_COLUMN, DOUBLE_SV_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.DOUBLE);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
- assertEquals(doubleValues.getLeft()[i], Math.max(_intSVValues[i], _doubleSVValues[i]));
+ assertEquals(doubleValues[i], Math.max(_intSVValues[i], _doubleSVValues[i]));
}
- assertEquals(doubleValues.getRight(), null);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ testNullBitmap(transformFunction, null);
}
@Test
public void testGreatestTransformFunctionNullColumn() {
- TransformFunction transformFunction =
- testGreatestPreconditions(String.format("greatest(%s, null, %s)", INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
+ TransformFunction transformFunction = testGreatestPreconditionsNullHandlingEnabled(
+ String.format("greatest(%s, null, %s)", INT_SV_NULL_COLUMN, DOUBLE_SV_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.DOUBLE);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
- assertEquals(doubleValues.getLeft()[i], Math.max(_intSVValues[i], _doubleSVValues[i]));
+ assertEquals(doubleValues[i], Math.max(_intSVValues[i], _doubleSVValues[i]));
} else {
- assertEquals(doubleValues.getLeft()[i], _doubleSVValues[i]);
+ assertEquals(doubleValues[i], _doubleSVValues[i]);
}
}
- assertEquals(doubleValues.getRight(), null);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), null);
+ testNullBitmap(transformFunction, null);
}
@Test
public void testGreatestTransformFunctionAllNulls() {
- TransformFunction transformFunction = testGreatestPreconditions(String.format("greatest(null, null, null)"));
+ TransformFunction transformFunction =
+ testGreatestPreconditionsNullHandlingEnabled(String.format("greatest(null, null, null)"));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.UNKNOWN);
- Pair<double[], RoaringBitmap> doubleValues = transformFunction.transformToDoubleValuesSVWithNull(_projectionBlock);
+ double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock);
RoaringBitmap expectedNull = new RoaringBitmap();
expectedNull.add(0L, NUM_ROWS);
- assertEquals(doubleValues.getRight(), expectedNull);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNull);
+ testNullBitmap(transformFunction, expectedNull);
}
@Test
public void testGreatestTransformFunctionPartialAllNulls() {
- TransformFunction transformFunction = testGreatestPreconditions(
+ TransformFunction transformFunction = testGreatestPreconditionsNullHandlingEnabled(
String.format("greatest(%s, %s, %s)", INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN, INT_SV_NULL_COLUMN));
assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.INT);
- Pair<int[], RoaringBitmap> intValues = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock);
RoaringBitmap expectedNull = new RoaringBitmap();
for (int i = 0; i < NUM_ROWS; i++) {
if (i % 2 == 0) {
- assertEquals(intValues.getLeft()[i], _intSVValues[i]);
+ assertEquals(intValues[i], _intSVValues[i]);
} else {
expectedNull.add(i);
}
}
- assertEquals(intValues.getRight(), expectedNull);
- assertEquals(transformFunction.getNullBitmap(_projectionBlock), expectedNull);
+ testNullBitmap(transformFunction, expectedNull);
}
@Test
@@ -511,4 +503,20 @@ public class TupleSelectionTransformFunctionsTest extends BaseTransformFunctionT
assertEquals(transformFunction.getName(), TransformFunctionType.GREATEST.getName());
return transformFunction;
}
+
+ private TransformFunction testLeastPreconditionsNullHandlingEnabled(String expressionStr) {
+ ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
+ TransformFunction transformFunction = TransformFunctionFactory.getNullHandlingEnabled(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof LeastTransformFunction);
+ assertEquals(transformFunction.getName(), TransformFunctionType.LEAST.getName());
+ return transformFunction;
+ }
+
+ private TransformFunction testGreatestPreconditionsNullHandlingEnabled(String expressionStr) {
+ ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
+ TransformFunction transformFunction = TransformFunctionFactory.getNullHandlingEnabled(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof GreatestTransformFunction);
+ assertEquals(transformFunction.getName(), TransformFunctionType.GREATEST.getName());
+ return transformFunction;
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
index ad5ba94b39..af54abf9a5 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/NullHandlingEnabledQueriesTest.java
@@ -446,4 +446,22 @@ public class NullHandlingEnabledQueriesTest extends BaseQueriesTest {
assertArrayEquals(rows.get(1), new Object[]{(long) NUM_OF_SEGMENT_COPIES, 3});
assertArrayEquals(rows.get(2), new Object[]{(long) NUM_OF_SEGMENT_COPIES, 2});
}
+
+ @Test
+ public void testNestedCaseTransformFunction()
+ throws Exception {
+ initializeRows();
+ insertRow(null);
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
+ Schema schema = new Schema.SchemaBuilder().addSingleValueDimension(COLUMN1, FieldSpec.DataType.INT).build();
+ setUpSegments(tableConfig, schema);
+ String query =
+ String.format("SELECT (CASE WHEN %s = -2147483648 THEN 1 ELSE 2 END) + 0 FROM testTable", COLUMN1);
+
+ BrokerResponseNative brokerResponse = getBrokerResponse(query, QUERY_OPTIONS);
+
+ ResultTable resultTable = brokerResponse.getResultTable();
+ List<Object[]> rows = resultTable.getRows();
+ assertArrayEquals(rows.get(0), new Object[]{(double) 2});
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org