You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2026/04/26 02:50:57 UTC
(pinot) branch master updated: Support MV BIG_DECIMAL and BYTES (#18326)
This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new aad170afaa5 Support MV BIG_DECIMAL and BYTES (#18326)
aad170afaa5 is described below
commit aad170afaa58a64d8d3ca3f4c6aba207beca1109
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Sat Apr 25 19:50:51 2026 -0700
Support MV BIG_DECIMAL and BYTES (#18326)
Adds end-to-end multi-value BIG_DECIMAL support: schema, segment
creation (dict/raw/fwd index/stats), block val-sets, transform functions,
aggregation (ARRAY_AGG, distinct aggregate), data block/table serde,
response encoders, and integration tests. Wires the same plumbing for
MV BYTES where it was missing so the two MV types are symmetric.
Also:
- Refactor ExprMinMax wrappers: dispatch on stored type, remove
redundant isSingleValue plumbing, fix BYTES_ARRAY getField path.
- Make DataSchema BYTES_ARRAY symmetric with BYTES across
convert / toExternal / format.
- ArrayListUtils: zero-copy fast-path for toBytesArray, new
toBigDecimalArray.
- TypeUtils.convert: MSE backward-compat downcast BIG_DECIMAL_ARRAY ->
DOUBLE_ARRAY (until wire protocol supports BIG_DECIMAL_ARRAY), add
asserts on fallthrough types, handle empty ObjectArrayList.
- Normalize case ordering so BIG_DECIMAL sits immediately after DOUBLE
across switches touched by this change.
---
.../pinot/common/datablock/BaseDataBlock.java | 40 +++
.../apache/pinot/common/datablock/DataBlock.java | 4 +
.../pinot/common/datablock/DataBlockEquals.java | 25 ++
.../apache/pinot/common/datatable/DataTable.java | 4 +
.../pinot/common/datatable/DataTableImplV4.java | 26 ++
.../pinot/common/function/FunctionUtils.java | 10 +
.../scalar/array/ArrayLengthScalarFunction.java | 15 +
.../scalar/array/ArraysOverlapScalarFunction.java | 42 +++
.../response/encoder/ArrowResponseEncoder.java | 15 +-
.../response/encoder/JsonResponseEncoder.java | 12 +-
.../apache/pinot/common/utils/ArrayListUtils.java | 32 +++
.../org/apache/pinot/common/utils/DataSchema.java | 134 ++++++---
.../apache/pinot/common/utils/PinotDataType.java | 47 +++-
pinot-common/src/main/proto/expressions.proto | 6 +
.../pinot/common/utils/ArrayListUtilsTest.java | 45 +++
.../org/apache/pinot/core/common/BlockValSet.java | 7 +
.../apache/pinot/core/common/DataBlockCache.java | 18 ++
.../org/apache/pinot/core/common/DataFetcher.java | 27 ++
.../apache/pinot/core/common/ObjectSerDeUtils.java | 141 +++++++++-
.../core/common/RowBasedBlockValueFetcher.java | 34 +++
.../core/common/datablock/DataBlockBuilder.java | 79 +++++-
.../common/datatable/BaseDataTableBuilder.java | 27 ++
.../core/common/datatable/DataTableBuilder.java | 6 +-
.../blocks/results/AggregationResultsBlock.java | 18 +-
.../blocks/results/GroupByResultsBlock.java | 17 ++
.../dociditerators/MVScanDocIdIterator.java | 15 +
.../core/operator/docvalsets/DataBlockValSet.java | 5 +
.../docvalsets/FilteredDataBlockValSet.java | 5 +
.../docvalsets/FilteredRowBasedBlockValSet.java | 5 +
.../operator/docvalsets/ProjectionBlockValSet.java | 8 +
.../operator/docvalsets/RowBasedBlockValSet.java | 62 ++++-
.../operator/docvalsets/TransformBlockValSet.java | 20 ++
.../BaseDictionaryBasedPredicateEvaluator.java | 5 +
.../BaseRawValueBasedPredicateEvaluator.java | 20 ++
.../filter/predicate/PredicateEvaluator.java | 9 +
.../query/NonScanBasedAggregationOperator.java | 8 +
.../function/ArrayLengthTransformFunction.java | 13 +
.../function/ArrayLiteralTransformFunction.java | 6 +
.../transform/function/BaseTransformFunction.java | 100 ++++++-
.../transform/function/CastTransformFunction.java | 27 +-
.../function/GenerateArrayTransformFunction.java | 5 +
.../function/IdentifierTransformFunction.java | 5 +
.../function/LiteralTransformFunction.java | 5 +
.../function/ScalarTransformFunctionWrapper.java | 6 +
.../transform/function/TransformFunction.java | 5 +
.../function/AggregationFunctionFactory.java | 12 +
.../function/AggregationFunctionUtils.java | 8 +
.../BaseDistinctAggregateAggregationFunction.java | 130 +++++++++
.../ParentExprMinMaxAggregationFunction.java | 186 ++-----------
.../function/array/ArrayAggBigDecimalFunction.java | 85 ++++++
.../function/array/ArrayAggBytesFunction.java | 90 ++++++
.../array/ArrayAggDistinctBigDecimalFunction.java | 86 ++++++
.../array/ArrayAggDistinctBytesFunction.java | 92 ++++++
.../array/BaseArrayAggBigDecimalFunction.java | 116 ++++++++
.../function/array/BaseArrayAggBytesFunction.java | 108 ++++++++
.../ParentAggregationFunctionResultObject.java | 4 +-
.../ExprMinMaxMeasuringValSetWrapper.java | 62 +++--
.../utils/exprminmax/ExprMinMaxObject.java | 27 +-
.../ExprMinMaxProjectionValSetWrapper.java | 20 +-
.../utils/exprminmax/ExprMinMaxWrapperValSet.java | 135 +++++----
.../distinct/raw/BigDecimalDistinctExecutor.java | 27 +-
.../core/query/reduce/GroupByDataTableReducer.java | 10 +-
.../query/selection/SelectionOperatorUtils.java | 12 +
.../rewriter/ParentAggregationResultRewriter.java | 5 +-
.../pinot/core/util/DataBlockExtractUtils.java | 4 +
.../org/apache/pinot/core/util/GapfillUtils.java | 3 +
.../pinot/core/common/SyntheticBlockValSets.java | 5 +
.../common/datablock/DataBlockBuilderTest.java | 75 ++++-
.../pinot/core/common/datablock/DataBlockTest.java | 49 ++--
.../core/common/datablock/DataBlockTestUtils.java | 30 +-
.../core/common/datatable/DataTableSerDeTest.java | 28 +-
.../DateTimeConversionTransformFunctionTest.java | 5 +
.../aggregation/function/ArrayAggFunctionTest.java | 26 ++
.../DistinctCountAggregationFunctionTest.java | 109 ++++++++
.../apache/pinot/core/util/SchemaUtilsTest.java | 29 +-
.../apache/pinot/queries/DistinctQueriesTest.java | 49 +++-
.../org/apache/pinot/queries/ExprMinMaxTest.java | 92 +++---
.../pinot/integration/tests/ClusterTest.java | 7 +-
.../pinot/integration/tests/custom/ArrayTest.java | 182 ++++++------
.../tests/custom/BigDecimalTypeTest.java | 307 +++++++++++++++++++++
.../integration/tests/custom/BytesMvTypeTest.java | 194 +++++++++++++
.../integration/tests/custom/BytesTypeTest.java | 15 +-
.../pinot/perf/BenchmarkScanDocIdIterators.java | 5 +
.../BenchmarkDistinctCountHLLThreshold.java | 5 +
.../planner/logical/RelToPlanNodeConverter.java | 7 +-
.../query/planner/logical/RexExpressionUtils.java | 11 -
.../physical/v2/PRelToPlanNodeConverter.java | 7 +-
.../serde/ProtoExpressionToRexExpression.java | 21 ++
.../serde/RexExpressionToProtoExpression.java | 20 ++
.../query/runtime/operator/utils/TypeUtils.java | 67 ++++-
.../impl/VarByteChunkForwardIndexWriter.java | 5 +
.../impl/VarByteChunkForwardIndexWriterV4.java | 5 +
.../local/io/writer/impl/VarByteChunkWriter.java | 2 +
.../creator/impl/SegmentDictionaryCreator.java | 13 +-
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 6 +
.../BigDecimalColumnPreIndexStatsCollector.java | 19 +-
.../index/forward/ForwardIndexCreatorFactory.java | 1 +
.../segment/index/loader/ForwardIndexHandler.java | 48 ++--
...IndexAndDictionaryBasedForwardIndexCreator.java | 26 +-
.../ColumnMinMaxValueGenerator.java | 27 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 50 +++-
.../forward/VarByteChunkForwardIndexReaderV4.java | 11 +
.../forward/VarByteChunkMVForwardIndexReader.java | 11 +
.../segment/readers/DefaultValueColumnReader.java | 22 ++
.../segment/readers/PinotSegmentColumnReader.java | 31 +++
.../readers/PinotSegmentColumnReaderImpl.java | 13 +
.../pinot/segment/local/utils/ArraySerDeUtils.java | 77 ++++--
.../pinot/segment/local/utils/SchemaUtils.java | 2 -
.../impl/ColumnarRowMajorEquivalenceTest.java | 1 +
.../impl/ColumnarSegmentBuildingTestBase.java | 10 +
.../AbstractColumnStatisticsCollectorTest.java | 32 ++-
.../impl/stats/EmptyColumnStatisticsTest.java | 4 +-
.../stats/NoDictColumnStatisticsCollectorTest.java | 4 +-
.../MultiValueVarByteRawIndexCreatorTest.java | 56 ++++
.../segment/readers/InMemoryColumnReader.java | 10 +
.../readers/PinotSegmentColumnReaderImplTest.java | 5 +
.../pinot/segment/spi/index/IndexCreator.java | 6 +
.../spi/index/creator/ForwardIndexCreator.java | 30 ++
.../spi/index/mutable/MutableForwardIndex.java | 21 ++
.../spi/index/reader/ForwardIndexReader.java | 155 +++++++++++
.../pinot/spi/data/readers/ColumnReader.java | 6 +
.../org/apache/pinot/spi/utils/ArrayCopyUtils.java | 57 ++++
.../apache/pinot/spi/utils/CommonConstants.java | 6 +-
123 files changed, 3821 insertions(+), 683 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java b/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
index 6e37f785d1b..136532e5e2b 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datablock/BaseDataBlock.java
@@ -288,6 +288,26 @@ public abstract class BaseDataBlock implements DataBlock {
return doubles;
}
+ @Override
+ public BigDecimal[] getBigDecimalArray(int rowId, int colId) {
+ int offsetInFixed = getOffsetInFixedBuffer(rowId, colId);
+ int size = _fixedSizeData.getInt(offsetInFixed + 4);
+ int offsetInVar = _fixedSizeData.getInt(offsetInFixed);
+
+ BigDecimal[] bigDecimals = new BigDecimal[size];
+ try (PinotInputStream stream = _variableSizeData.openInputStream(offsetInVar)) {
+ for (int i = 0; i < size; i++) {
+ int byteLength = stream.readInt();
+ byte[] bytes = new byte[byteLength];
+ stream.readFully(bytes);
+ bigDecimals[i] = BigDecimalUtils.deserialize(bytes);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return bigDecimals;
+ }
+
@Override
public String[] getStringArray(int rowId, int colId) {
int offsetInFixed = getOffsetInFixedBuffer(rowId, colId);
@@ -305,6 +325,26 @@ public abstract class BaseDataBlock implements DataBlock {
return strings;
}
+ @Override
+ public ByteArray[] getBytesArray(int rowId, int colId) {
+ int offsetInFixed = getOffsetInFixedBuffer(rowId, colId);
+ int size = _fixedSizeData.getInt(offsetInFixed + 4);
+ int offsetInVar = _fixedSizeData.getInt(offsetInFixed);
+
+ ByteArray[] bytes = new ByteArray[size];
+ try (PinotInputStream stream = _variableSizeData.openInputStream(offsetInVar)) {
+ for (int i = 0; i < size; i++) {
+ int byteLength = stream.readInt();
+ byte[] value = new byte[byteLength];
+ stream.readFully(value);
+ bytes[i] = new ByteArray(value);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return bytes;
+ }
+
@Override
public Map<String, Object> getMap(int rowId, int colId) {
int offsetInFixed = getOffsetInFixedBuffer(rowId, colId);
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlock.java b/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlock.java
index 97ffd602fd5..3699d28b312 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlock.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlock.java
@@ -83,8 +83,12 @@ public interface DataBlock {
double[] getDoubleArray(int rowId, int colId);
+ BigDecimal[] getBigDecimalArray(int rowId, int colId);
+
String[] getStringArray(int rowId, int colId);
+ ByteArray[] getBytesArray(int rowId, int colId);
+
Map<String, Object> getMap(int rowId, int colId);
@Nullable
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlockEquals.java b/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlockEquals.java
index 2f8a2e90549..d75f2b38ff4 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlockEquals.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datablock/DataBlockEquals.java
@@ -396,6 +396,19 @@ public class DataBlockEquals {
}
}
break;
+ case BIG_DECIMAL_ARRAY:
+ for (int did = 0; did < numRows; did++) {
+ if (!Arrays.equals(left.getBigDecimalArray(did, colId), right.getBigDecimalArray(did, colId))) {
+ if (_failOnFalse) {
+ throw new IllegalArgumentException("Different values for " + colTypes[colId]
+ + " column: " + colNames[colId]
+ + " left: " + Arrays.toString(left.getBigDecimalArray(did, colId))
+ + " right: " + Arrays.toString(right.getBigDecimalArray(did, colId)));
+ }
+ return false;
+ }
+ }
+ break;
case STRING_ARRAY:
for (int did = 0; did < numRows; did++) {
if (!Arrays.equals(left.getStringArray(did, colId), right.getStringArray(did, colId))) {
@@ -410,6 +423,18 @@ public class DataBlockEquals {
}
break;
case BYTES_ARRAY:
+ for (int did = 0; did < numRows; did++) {
+ if (!Arrays.equals(left.getBytesArray(did, colId), right.getBytesArray(did, colId))) {
+ if (_failOnFalse) {
+ throw new IllegalArgumentException("Different values for " + colTypes[colId]
+ + " column: " + colNames[colId]
+ + " left: " + Arrays.toString(left.getBytesArray(did, colId))
+ + " right: " + Arrays.toString(right.getBytesArray(did, colId)));
+ }
+ return false;
+ }
+ }
+ break;
case BOOLEAN_ARRAY:
case UNKNOWN:
throw new UnsupportedOperationException("Check how to read " + colTypes[colId] + " from data block");
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTable.java b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTable.java
index 5c7cd83ff29..bf3f2f86117 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTable.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTable.java
@@ -77,8 +77,12 @@ public interface DataTable {
double[] getDoubleArray(int rowId, int colId);
+ BigDecimal[] getBigDecimalArray(int rowId, int colId);
+
String[] getStringArray(int rowId, int colId);
+ ByteArray[] getBytesArray(int rowId, int colId);
+
@Nullable
Map<String, Object> getMap(int rowId, int colId);
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
index 64bda5d0f04..2801c2b0426 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/datatable/DataTableImplV4.java
@@ -308,6 +308,19 @@ public class DataTableImplV4 implements DataTable {
return doubles;
}
+ @Override
+ public BigDecimal[] getBigDecimalArray(int rowId, int colId) {
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
+ BigDecimal[] bigDecimals = new BigDecimal[length];
+ for (int i = 0; i < length; i++) {
+ int byteLength = _variableSizeData.getInt();
+ byte[] bytes = new byte[byteLength];
+ _variableSizeData.get(bytes);
+ bigDecimals[i] = BigDecimalUtils.deserialize(bytes);
+ }
+ return bigDecimals;
+ }
+
@Override
public String[] getStringArray(int rowId, int colId) {
int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
@@ -318,6 +331,19 @@ public class DataTableImplV4 implements DataTable {
return strings;
}
+ @Override
+ public ByteArray[] getBytesArray(int rowId, int colId) {
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
+ ByteArray[] bytes = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ int byteLength = _variableSizeData.getInt();
+ byte[] value = new byte[byteLength];
+ _variableSizeData.get(value);
+ bytes[i] = new ByteArray(value);
+ }
+ return bytes;
+ }
+
@Nullable
@Override
public Map<String, Object> getMap(int rowId, int colId) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java
index e4f6760929d..60ee07a76ab 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java
@@ -56,7 +56,9 @@ public class FunctionUtils {
put(long[].class, PinotDataType.PRIMITIVE_LONG_ARRAY);
put(float[].class, PinotDataType.PRIMITIVE_FLOAT_ARRAY);
put(double[].class, PinotDataType.PRIMITIVE_DOUBLE_ARRAY);
+ put(BigDecimal[].class, PinotDataType.BIG_DECIMAL_ARRAY);
put(String[].class, PinotDataType.STRING_ARRAY);
+ put(byte[][].class, PinotDataType.BYTES_ARRAY);
put(Map.class, PinotDataType.MAP);
put(Object.class, PinotDataType.OBJECT);
}};
@@ -83,7 +85,9 @@ public class FunctionUtils {
put(Float[].class, PinotDataType.FLOAT_ARRAY);
put(double[].class, PinotDataType.PRIMITIVE_DOUBLE_ARRAY);
put(Double[].class, PinotDataType.DOUBLE_ARRAY);
+ put(BigDecimal[].class, PinotDataType.BIG_DECIMAL_ARRAY);
put(String[].class, PinotDataType.STRING_ARRAY);
+ put(byte[][].class, PinotDataType.BYTES_ARRAY);
put(Object.class, PinotDataType.OBJECT);
put(Object[].class, PinotDataType.OBJECT_ARRAY);
}};
@@ -107,7 +111,9 @@ public class FunctionUtils {
put(long[].class, DataType.LONG);
put(float[].class, DataType.FLOAT);
put(double[].class, DataType.DOUBLE);
+ put(BigDecimal[].class, DataType.BIG_DECIMAL);
put(String[].class, DataType.STRING);
+ put(byte[][].class, DataType.BYTES);
}};
private static final Map<Class<?>, ColumnDataType> COLUMN_DATA_TYPE_MAP = new HashMap<>() {{
@@ -129,7 +135,9 @@ public class FunctionUtils {
put(long[].class, ColumnDataType.LONG_ARRAY);
put(float[].class, ColumnDataType.FLOAT_ARRAY);
put(double[].class, ColumnDataType.DOUBLE_ARRAY);
+ put(BigDecimal[].class, ColumnDataType.BIG_DECIMAL_ARRAY);
put(String[].class, ColumnDataType.STRING_ARRAY);
+ put(byte[][].class, ColumnDataType.BYTES_ARRAY);
put(Object.class, ColumnDataType.OBJECT);
}};
@@ -207,6 +215,8 @@ public class FunctionUtils {
return typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.FLOAT), -1);
case DOUBLE_ARRAY:
return typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.DOUBLE), -1);
+ case BIG_DECIMAL_ARRAY:
+ return typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.DECIMAL), -1);
case BOOLEAN_ARRAY:
return typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.BOOLEAN), -1);
case TIMESTAMP_ARRAY:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArrayLengthScalarFunction.java b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArrayLengthScalarFunction.java
index 45c8319c608..60fab79baa3 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArrayLengthScalarFunction.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArrayLengthScalarFunction.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.common.function.scalar.array;
+import java.math.BigDecimal;
import java.util.EnumMap;
import java.util.Map;
import java.util.Set;
@@ -49,9 +50,15 @@ public class ArrayLengthScalarFunction implements PinotScalarFunction {
TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.DOUBLE_ARRAY,
new FunctionInfo(ArrayLengthScalarFunction.class.getMethod("arrayLength", double[].class),
ArrayLengthScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.BIG_DECIMAL_ARRAY,
+ new FunctionInfo(ArrayLengthScalarFunction.class.getMethod("arrayLength", BigDecimal[].class),
+ ArrayLengthScalarFunction.class, false));
TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.STRING_ARRAY,
new FunctionInfo(ArrayLengthScalarFunction.class.getMethod("arrayLength", String[].class),
ArrayLengthScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.BYTES_ARRAY,
+ new FunctionInfo(ArrayLengthScalarFunction.class.getMethod("arrayLength", byte[][].class),
+ ArrayLengthScalarFunction.class, false));
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
@@ -109,7 +116,15 @@ public class ArrayLengthScalarFunction implements PinotScalarFunction {
return array.length;
}
+ public static int arrayLength(BigDecimal[] array) {
+ return array.length;
+ }
+
public static int arrayLength(String[] array) {
return array.length;
}
+
+ public static int arrayLength(byte[][] array) {
+ return array.length;
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArraysOverlapScalarFunction.java b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArraysOverlapScalarFunction.java
index f8c46b69026..029d0b3f45a 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArraysOverlapScalarFunction.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/array/ArraysOverlapScalarFunction.java
@@ -23,6 +23,7 @@ import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import java.math.BigDecimal;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
@@ -36,6 +37,7 @@ import org.apache.pinot.common.function.PinotScalarFunction;
import org.apache.pinot.common.function.sql.PinotSqlFunction;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.spi.annotations.ScalarFunction;
+import org.apache.pinot.spi.utils.ByteArray;
@ScalarFunction(names = {"ARRAYS_OVERLAP"})
@@ -58,9 +60,16 @@ public class ArraysOverlapScalarFunction implements PinotScalarFunction {
TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.DOUBLE_ARRAY,
new FunctionInfo(ArraysOverlapScalarFunction.class.getMethod("arraysOverlap", double[].class, double[].class),
ArraysOverlapScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.BIG_DECIMAL_ARRAY,
+ new FunctionInfo(
+ ArraysOverlapScalarFunction.class.getMethod("arraysOverlap", BigDecimal[].class, BigDecimal[].class),
+ ArraysOverlapScalarFunction.class, false));
TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.STRING_ARRAY,
new FunctionInfo(ArraysOverlapScalarFunction.class.getMethod("arraysOverlap", String[].class, String[].class),
ArraysOverlapScalarFunction.class, false));
+ TYPE_FUNCTION_INFO_MAP.put(DataSchema.ColumnDataType.BYTES_ARRAY,
+ new FunctionInfo(ArraysOverlapScalarFunction.class.getMethod("arraysOverlap", byte[][].class, byte[][].class),
+ ArraysOverlapScalarFunction.class, false));
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
@@ -159,6 +168,31 @@ public class ArraysOverlapScalarFunction implements PinotScalarFunction {
return false;
}
+ private static boolean overlapBigDecimals(BigDecimal[] small, BigDecimal[] large) {
+ ObjectOpenHashSet<BigDecimal> set = new ObjectOpenHashSet<>(small);
+ for (BigDecimal v : large) {
+ if (set.contains(v)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /// Wraps each raw `byte[]` into a [ByteArray] so the hash set can use content-based equality; `byte[]`
+ /// itself only has reference equality.
+ private static boolean overlapBytes(byte[][] small, byte[][] large) {
+ ObjectOpenHashSet<ByteArray> set = new ObjectOpenHashSet<>(small.length);
+ for (byte[] v : small) {
+ set.add(new ByteArray(v));
+ }
+ for (byte[] v : large) {
+ if (set.contains(new ByteArray(v))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static boolean arraysOverlap(int[] array1, int[] array2) {
return array1.length <= array2.length ? overlapInts(array1, array2) : overlapInts(array2, array1);
}
@@ -175,7 +209,15 @@ public class ArraysOverlapScalarFunction implements PinotScalarFunction {
return array1.length <= array2.length ? overlapDoubles(array1, array2) : overlapDoubles(array2, array1);
}
+ public static boolean arraysOverlap(BigDecimal[] array1, BigDecimal[] array2) {
+ return array1.length <= array2.length ? overlapBigDecimals(array1, array2) : overlapBigDecimals(array2, array1);
+ }
+
public static boolean arraysOverlap(String[] array1, String[] array2) {
return array1.length <= array2.length ? overlapStrings(array1, array2) : overlapStrings(array2, array1);
}
+
+ public static boolean arraysOverlap(byte[][] array1, byte[][] array2) {
+ return array1.length <= array2.length ? overlapBytes(array1, array2) : overlapBytes(array2, array1);
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
index a17eeb70537..ce860dbecc4 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
@@ -103,11 +103,11 @@ public class ArrowResponseEncoder implements ResponseEncoder {
new Field(colName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null);
vector = new Float8Vector(colName, ALLOCATOR);
break;
+ case BIG_DECIMAL:
case TIMESTAMP:
case STRING:
- case BYTES:
- case BIG_DECIMAL:
case JSON:
+ case BYTES:
case OBJECT:
field = new Field(colName, FieldType.nullable(new ArrowType.Utf8()), null);
vector = new VarCharVector(colName, ALLOCATOR);
@@ -178,6 +178,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
// Initialize its children from the defined field.
vector.initializeChildrenFromFields(children);
break;
+ case BIG_DECIMAL_ARRAY:
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
@@ -229,11 +230,11 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case DOUBLE:
((Float8Vector) vector).setSafe(rowIndex, ((Number) value).doubleValue());
break;
+ case BIG_DECIMAL:
case TIMESTAMP:
case STRING:
- case BYTES:
- case BIG_DECIMAL:
case JSON:
+ case BYTES:
case OBJECT:
byte[] bytes = ((String) value).getBytes(StandardCharsets.UTF_8);
((VarCharVector) vector).setSafe(rowIndex, bytes);
@@ -341,6 +342,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
// Finalize the list entry by indicating the number of elements added.
doubleArrayVector.endValue(rowIndex, doubleArray.length);
break;
+ case BIG_DECIMAL_ARRAY:
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
@@ -405,11 +407,11 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case BOOLEAN:
row[col] = ((BitVector) vector).get(i) == 1;
break;
+ case BIG_DECIMAL:
case TIMESTAMP:
case STRING:
- case BYTES:
- case BIG_DECIMAL:
case JSON:
+ case BYTES:
case OBJECT:
row[col] = new String(((VarCharVector) vector).get(i), StandardCharsets.UTF_8);
break;
@@ -464,6 +466,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
}
row[col] = doubleArray;
break;
+ case BIG_DECIMAL_ARRAY:
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
index 0b260779e16..0f82daab9e9 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
@@ -190,8 +190,9 @@ public class JsonResponseEncoder implements ResponseEncoder {
doubleArray[k] = jsonValue.get(k).asDouble();
}
return doubleArray;
- case STRING_ARRAY:
+ case BIG_DECIMAL_ARRAY:
case TIMESTAMP_ARRAY:
+ case STRING_ARRAY:
case BYTES_ARRAY:
String[] stringArray = new String[jsonValue.size()];
for (int k = 0; k < jsonValue.size(); k++) {
@@ -203,8 +204,7 @@ public class JsonResponseEncoder implements ResponseEncoder {
}
}
- private static Object extractValue(DataSchema.ColumnDataType columnDataType, JsonNode jsonValue)
- throws IOException {
+ private static Object extractValue(DataSchema.ColumnDataType columnDataType, JsonNode jsonValue) {
if (jsonValue.isNull()) {
return null;
}
@@ -219,11 +219,11 @@ public class JsonResponseEncoder implements ResponseEncoder {
return Double.valueOf(jsonValue.asDouble()).floatValue();
case DOUBLE:
return jsonValue.asDouble();
- case STRING:
- case BYTES:
+ case BIG_DECIMAL:
case TIMESTAMP:
+ case STRING:
case JSON:
- case BIG_DECIMAL:
+ case BYTES:
case OBJECT:
return jsonValue.textValue();
case UNKNOWN:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/ArrayListUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/ArrayListUtils.java
index f74f3265559..9f90e718b61 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/ArrayListUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/ArrayListUtils.java
@@ -23,6 +23,8 @@ import it.unimi.dsi.fastutil.floats.FloatArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import java.math.BigDecimal;
+import org.apache.pinot.spi.utils.ByteArray;
/**
@@ -111,6 +113,21 @@ public class ArrayListUtils {
: doubleArrayList.toDoubleArray();
}
+ /**
+ * Convert the given {@link ObjectArrayList} to a BigDecimal array. Mirrors {@link #toStringArray}: returns the
+ * backing array reference when possible, otherwise copies elements into a new array.
+ */
+ public static BigDecimal[] toBigDecimalArray(ObjectArrayList<BigDecimal> bigDecimalArrayList) {
+ Object elements = bigDecimalArrayList.elements();
+ if (elements instanceof BigDecimal[]) {
+ BigDecimal[] bigDecimalArrayListElements = (BigDecimal[]) elements;
+ if (bigDecimalArrayListElements.length == bigDecimalArrayList.size()) {
+ return bigDecimalArrayListElements;
+ }
+ }
+ return bigDecimalArrayList.toArray(new BigDecimal[0]);
+ }
+
/**
* Convert the given {@link ObjectArrayList} to a string array.
* The method {@link ObjectArrayList#elements()} could return either Object[] or String[]. The casting to String[]
@@ -136,4 +153,19 @@ public class ArrayListUtils {
}
return stringArrayList.toArray(new String[0]);
}
+
+ /**
+ * Convert the given {@link ObjectArrayList} to a ByteArray array. Mirrors {@link #toStringArray}: returns the
+ * backing array reference when possible, otherwise copies elements into a new array.
+ */
+ public static ByteArray[] toBytesArray(ObjectArrayList<ByteArray> bytesArrayList) {
+ Object elements = bytesArrayList.elements();
+ if (elements instanceof ByteArray[]) {
+ ByteArray[] bytesArrayListElements = (ByteArray[]) elements;
+ if (bytesArrayListElements.length == bytesArrayList.size()) {
+ return bytesArrayListElements;
+ }
+ }
+ return bytesArrayList.toArray(new ByteArray[0]);
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
index 405ab993782..be5768a3d41 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -332,6 +332,12 @@ public class DataSchema {
return typeFactory.createArrayType(DOUBLE.toType(typeFactory), -1);
}
},
+ BIG_DECIMAL_ARRAY(NullValuePlaceHolder.BIG_DECIMAL_ARRAY) {
+ @Override
+ public RelDataType toType(RelDataTypeFactory typeFactory) {
+ return typeFactory.createArrayType(BIG_DECIMAL.toType(typeFactory), -1);
+ }
+ },
BOOLEAN_ARRAY(INT_ARRAY, NullValuePlaceHolder.INT_ARRAY) {
@Override
public RelDataType toType(RelDataTypeFactory typeFactory) {
@@ -350,7 +356,7 @@ public class DataSchema {
return typeFactory.createArrayType(STRING.toType(typeFactory), -1);
}
},
- BYTES_ARRAY(NullValuePlaceHolder.BYTES_ARRAY) {
+ BYTES_ARRAY(NullValuePlaceHolder.INTERNAL_BYTES_ARRAY) {
@Override
public RelDataType toType(RelDataTypeFactory typeFactory) {
return typeFactory.createArrayType(BYTES.toType(typeFactory), -1);
@@ -366,10 +372,10 @@ public class DataSchema {
private static final EnumSet<ColumnDataType> NUMERIC_TYPES = EnumSet.of(INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL);
private static final EnumSet<ColumnDataType> INTEGRAL_TYPES = EnumSet.of(INT, LONG);
private static final EnumSet<ColumnDataType> ARRAY_TYPES =
- EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
- BYTES_ARRAY);
+ EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, BIG_DECIMAL_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
+ STRING_ARRAY, BYTES_ARRAY);
private static final EnumSet<ColumnDataType> NUMERIC_ARRAY_TYPES =
- EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY);
+ EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, BIG_DECIMAL_ARRAY);
private static final EnumSet<ColumnDataType> INTEGRAL_ARRAY_TYPES = EnumSet.of(INT_ARRAY, LONG_ARRAY);
// stored data type.
@@ -440,6 +446,7 @@ public class DataSchema {
case DOUBLE_ARRAY:
return DataType.DOUBLE;
case BIG_DECIMAL:
+ case BIG_DECIMAL_ARRAY:
return DataType.BIG_DECIMAL;
case BOOLEAN:
case BOOLEAN_ARRAY:
@@ -530,6 +537,7 @@ public class DataSchema {
* <li>BYTES: ByteArray -> byte[]</li>
* <li>BOOLEAN_ARRAY: int[] -> boolean[]</li>
* <li>TIMESTAMP_ARRAY: long[] -> Timestamp[]</li>
+ * <li>BYTES_ARRAY: ByteArray[] -> byte[][]</li>
* </ul>
*/
public Object toExternal(Object value) {
@@ -544,6 +552,8 @@ public class DataSchema {
return toBooleanArray((int[]) value);
case TIMESTAMP_ARRAY:
return toTimestampArray((long[]) value);
+ case BYTES_ARRAY:
+ return toBytesArray(value);
default:
return value;
}
@@ -582,14 +592,16 @@ public class DataSchema {
return toFloatArray(value);
case DOUBLE_ARRAY:
return toDoubleArray(value);
- case STRING_ARRAY:
- return toStringArray(value);
+ case BIG_DECIMAL_ARRAY:
+ return toBigDecimalArray(value);
case BOOLEAN_ARRAY:
return toBooleanArray(toIntArray(value));
case TIMESTAMP_ARRAY:
return toTimestampArray(toLongArray(value));
+ case STRING_ARRAY:
+ return toStringArray(value);
case BYTES_ARRAY:
- return (byte[][]) value;
+ return toBytesArray(value);
case UNKNOWN: // fall through
case OBJECT:
return (Serializable) value;
@@ -611,6 +623,8 @@ public class DataSchema {
return value.toString();
case BYTES:
return BytesUtils.toHexString((byte[]) value);
+ case BIG_DECIMAL_ARRAY:
+ return formatBigDecimalArray((BigDecimal[]) value);
case TIMESTAMP_ARRAY:
return formatTimestampArray((Timestamp[]) value);
case BYTES_ARRAY:
@@ -654,14 +668,16 @@ public class DataSchema {
return (float[]) value;
case DOUBLE_ARRAY:
return toDoubleArray(value);
- case STRING_ARRAY:
- return (String[]) value;
+ case BIG_DECIMAL_ARRAY:
+ return formatBigDecimalArray((BigDecimal[]) value);
case BOOLEAN_ARRAY:
return toBooleanArray((int[]) value);
case TIMESTAMP_ARRAY:
return formatTimestampArray((long[]) value);
+ case STRING_ARRAY:
+ return (String[]) value;
case BYTES_ARRAY:
- return (byte[][]) value;
+ return formatBytesArray((ByteArray[]) value);
default:
throw new IllegalStateException(String.format("Cannot convert and format: '%s' to type: %s", value, this));
}
@@ -687,6 +703,23 @@ public class DataSchema {
throw new IllegalStateException(String.format("Cannot convert: '%s' to int[]", value));
}
+ private static long[] toLongArray(Object value) {
+ if (value instanceof long[]) {
+ return (long[]) value;
+ } else if (value instanceof LongArrayList) {
+ // For FunnelCountAggregationFunction and ArrayAggregationFunction
+ return ArrayListUtils.toLongArray((LongArrayList) value);
+ } else {
+ int[] intValues = (int[]) value;
+ int length = intValues.length;
+ long[] longValues = new long[length];
+ for (int i = 0; i < length; i++) {
+ longValues[i] = intValues[i];
+ }
+ return longValues;
+ }
+ }
+
private static float[] toFloatArray(Object value) {
if (value instanceof float[]) {
return (float[]) value;
@@ -730,31 +763,23 @@ public class DataSchema {
}
}
- private static long[] toLongArray(Object value) {
- if (value instanceof long[]) {
- return (long[]) value;
- } else if (value instanceof LongArrayList) {
- // For FunnelCountAggregationFunction and ArrayAggregationFunction
- return ArrayListUtils.toLongArray((LongArrayList) value);
- } else {
- int[] intValues = (int[]) value;
- int length = intValues.length;
- long[] longValues = new long[length];
- for (int i = 0; i < length; i++) {
- longValues[i] = intValues[i];
- }
- return longValues;
+ private static BigDecimal[] toBigDecimalArray(Object value) {
+ if (value instanceof BigDecimal[]) {
+ return (BigDecimal[]) value;
+ } else if (value instanceof ObjectArrayList) {
+ // For ArrayAggregationFunction
+ return ArrayListUtils.toBigDecimalArray((ObjectArrayList<BigDecimal>) value);
}
+ throw new IllegalStateException(String.format("Cannot convert: '%s' to BigDecimal[]", value));
}
- private static String[] toStringArray(Object value) {
- if (value instanceof String[]) {
- return (String[]) value;
- } else if (value instanceof ObjectArrayList) {
- // For ArrayAggregationFunction
- return ArrayListUtils.toStringArray((ObjectArrayList<String>) value);
+ private static String[] formatBigDecimalArray(BigDecimal[] bigDecimalArray) {
+ int length = bigDecimalArray.length;
+ String[] formattedBigDecimalArray = new String[length];
+ for (int i = 0; i < length; i++) {
+ formattedBigDecimalArray[i] = bigDecimalArray[i].toPlainString();
}
- throw new IllegalStateException(String.format("Cannot convert: '%s' to String[]", value));
+ return formattedBigDecimalArray;
}
private static boolean[] toBooleanArray(int[] intArray) {
@@ -811,11 +836,52 @@ public class DataSchema {
return formattedTimestampArray;
}
- private static String[] formatBytesArray(byte[][] byteArray) {
+ private static String[] toStringArray(Object value) {
+ if (value instanceof String[]) {
+ return (String[]) value;
+ } else if (value instanceof ObjectArrayList) {
+ // For ArrayAggregationFunction
+ return ArrayListUtils.toStringArray((ObjectArrayList<String>) value);
+ }
+ throw new IllegalStateException(String.format("Cannot convert: '%s' to String[]", value));
+ }
+
+ private static byte[][] toBytesArray(Object value) {
+ if (value instanceof ByteArray[]) {
+ ByteArray[] wrapped = (ByteArray[]) value;
+ byte[][] raw = new byte[wrapped.length][];
+ for (int i = 0; i < wrapped.length; i++) {
+ raw[i] = wrapped[i].getBytes();
+ }
+ return raw;
+ }
+ if (value instanceof ObjectArrayList) {
+ // For ArrayAggregationFunction
+ ObjectArrayList<ByteArray> list = (ObjectArrayList<ByteArray>) value;
+ int size = list.size();
+ byte[][] raw = new byte[size][];
+ for (int i = 0; i < size; i++) {
+ raw[i] = list.get(i).getBytes();
+ }
+ return raw;
+ }
+ throw new IllegalStateException(String.format("Cannot convert: '%s' to byte[][]", value));
+ }
+
+ private static String[] formatBytesArray(byte[][] bytesArray) {
+ int length = bytesArray.length;
+ String[] formattedBytesArray = new String[length];
+ for (int i = 0; i < length; i++) {
+ formattedBytesArray[i] = BytesUtils.toHexString(bytesArray[i]);
+ }
+ return formattedBytesArray;
+ }
+
+ private static String[] formatBytesArray(ByteArray[] byteArray) {
int length = byteArray.length;
String[] formattedBytesArray = new String[length];
for (int i = 0; i < length; i++) {
- formattedBytesArray[i] = BytesUtils.toHexString(byteArray[i]);
+ formattedBytesArray[i] = byteArray[i].toHexString();
}
return formattedBytesArray;
}
@@ -865,6 +931,8 @@ public class DataSchema {
return FLOAT_ARRAY;
case DOUBLE:
return DOUBLE_ARRAY;
+ case BIG_DECIMAL:
+ return BIG_DECIMAL_ARRAY;
case BOOLEAN:
return BOOLEAN_ARRAY;
case TIMESTAMP:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
index c8cc2a242b8..3bda1b190f1 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/PinotDataType.java
@@ -935,6 +935,13 @@ public enum PinotDataType {
}
},
+ BIG_DECIMAL_ARRAY {
+ @Override
+ public BigDecimal[] convert(Object value, PinotDataType sourceType) {
+ return sourceType.toBigDecimalArray(value);
+ }
+ },
+
BOOLEAN_ARRAY {
@Override
public boolean[] convert(Object value, PinotDataType sourceType) {
@@ -1264,6 +1271,24 @@ public enum PinotDataType {
}
}
+ public BigDecimal[] toBigDecimalArray(Object value) {
+ if (value instanceof BigDecimal[]) {
+ return (BigDecimal[]) value;
+ }
+ if (isSingleValue()) {
+ return new BigDecimal[]{toBigDecimal(value)};
+ } else {
+ Object[] valueArray = toObjectArray(value);
+ int length = valueArray.length;
+ BigDecimal[] bigDecimalArray = new BigDecimal[length];
+ PinotDataType singleValueType = getSingleValueType();
+ for (int i = 0; i < length; i++) {
+ bigDecimalArray[i] = singleValueType.toBigDecimal(valueArray[i]);
+ }
+ return bigDecimalArray;
+ }
+ }
+
private static Object[] toObjectArray(Object array) {
if (array instanceof Collection) {
return ((Collection<?>) array).toArray();
@@ -1365,6 +1390,12 @@ public enum PinotDataType {
case PRIMITIVE_DOUBLE_ARRAY:
case DOUBLE_ARRAY:
return DOUBLE;
+ case BIG_DECIMAL_ARRAY:
+ return BIG_DECIMAL;
+ case BOOLEAN_ARRAY:
+ return BOOLEAN;
+ case TIMESTAMP_ARRAY:
+ return TIMESTAMP;
case STRING_ARRAY:
return STRING;
case BYTES_ARRAY:
@@ -1372,10 +1403,6 @@ public enum PinotDataType {
case OBJECT_ARRAY:
case COLLECTION:
return OBJECT;
- case BOOLEAN_ARRAY:
- return BOOLEAN;
- case TIMESTAMP_ARRAY:
- return TIMESTAMP;
default:
throw new IllegalStateException("There is no single-value type for " + this);
}
@@ -1437,6 +1464,9 @@ public enum PinotDataType {
if (cls == Double.class) {
return DOUBLE_ARRAY;
}
+ if (cls == BigDecimal.class) {
+ return BIG_DECIMAL_ARRAY;
+ }
if (cls == String.class) {
return STRING_ARRAY;
}
@@ -1493,10 +1523,7 @@ public enum PinotDataType {
case DOUBLE:
return fieldSpec.isSingleValueField() ? DOUBLE : DOUBLE_ARRAY;
case BIG_DECIMAL:
- if (fieldSpec.isSingleValueField()) {
- return BIG_DECIMAL;
- }
- throw new IllegalStateException("There is no multi-value type for BigDecimal");
+ return fieldSpec.isSingleValueField() ? BIG_DECIMAL : BIG_DECIMAL_ARRAY;
case BOOLEAN:
return fieldSpec.isSingleValueField() ? BOOLEAN : BOOLEAN_ARRAY;
case TIMESTAMP:
@@ -1557,8 +1584,12 @@ public enum PinotDataType {
return PRIMITIVE_FLOAT_ARRAY;
case DOUBLE_ARRAY:
return PRIMITIVE_DOUBLE_ARRAY;
+ case BIG_DECIMAL_ARRAY:
+ return BIG_DECIMAL_ARRAY;
case STRING_ARRAY:
return STRING_ARRAY;
+ case BYTES_ARRAY:
+ return BYTES_ARRAY;
default:
throw new IllegalStateException("Cannot convert ColumnDataType: " + columnDataType + " to PinotDataType");
}
diff --git a/pinot-common/src/main/proto/expressions.proto b/pinot-common/src/main/proto/expressions.proto
index e402bd97c89..eafb4601894 100644
--- a/pinot-common/src/main/proto/expressions.proto
+++ b/pinot-common/src/main/proto/expressions.proto
@@ -43,6 +43,7 @@ enum ColumnDataType {
BYTES_ARRAY = 18;
UNKNOWN = 19;
MAP = 20;
+ BIG_DECIMAL_ARRAY = 21;
}
message InputRef {
@@ -64,6 +65,7 @@ message Literal {
FloatArray floatArray = 11;
DoubleArray doubleArray = 12;
StringArray stringArray = 13;
+ BytesArray bytesArray = 14;
}
}
@@ -87,6 +89,10 @@ message StringArray {
repeated string values = 1;
}
+message BytesArray {
+ repeated bytes values = 1;
+}
+
message FunctionCall {
ColumnDataType dataType = 1;
string functionName = 2;
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/ArrayListUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/ArrayListUtilsTest.java
index 815114e899f..f4f9a72a78b 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/ArrayListUtilsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/ArrayListUtilsTest.java
@@ -23,6 +23,8 @@ import it.unimi.dsi.fastutil.floats.FloatArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import java.math.BigDecimal;
+import org.apache.pinot.spi.utils.ByteArray;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -121,6 +123,29 @@ public class ArrayListUtilsTest {
assertEquals(doubleArray[2], 3.0);
}
+ @Test
+ public void testToBigDecimalArray() {
+ // Test empty list
+ ObjectArrayList<BigDecimal> bigDecimalArrayList = new ObjectArrayList<>();
+ BigDecimal[] bigDecimalArray = ArrayListUtils.toBigDecimalArray(bigDecimalArrayList);
+ assertEquals(bigDecimalArray.length, 0);
+
+ // Test list with one element
+ bigDecimalArrayList.add(new BigDecimal("1.1"));
+ bigDecimalArray = ArrayListUtils.toBigDecimalArray(bigDecimalArrayList);
+ assertEquals(bigDecimalArray.length, 1);
+ assertEquals(bigDecimalArray[0], new BigDecimal("1.1"));
+
+ // Test list with multiple elements
+ bigDecimalArrayList.add(new BigDecimal("2.22"));
+ bigDecimalArrayList.add(new BigDecimal("3.333"));
+ bigDecimalArray = ArrayListUtils.toBigDecimalArray(bigDecimalArrayList);
+ assertEquals(bigDecimalArray.length, 3);
+ assertEquals(bigDecimalArray[0], new BigDecimal("1.1"));
+ assertEquals(bigDecimalArray[1], new BigDecimal("2.22"));
+ assertEquals(bigDecimalArray[2], new BigDecimal("3.333"));
+ }
+
@Test
public void testToStringArray() {
// Test empty list
@@ -143,4 +168,24 @@ public class ArrayListUtilsTest {
assertEquals(stringArray[1], "2");
assertEquals(stringArray[2], "3");
}
+
+ @Test
+ public void testToBytesArray() {
+ ObjectArrayList<ByteArray> bytesArrayList = new ObjectArrayList<>();
+ ByteArray[] bytesArray = ArrayListUtils.toBytesArray(bytesArrayList);
+ assertEquals(bytesArray.length, 0);
+
+ bytesArrayList.add(new ByteArray(new byte[]{0x01, 0x02}));
+ bytesArray = ArrayListUtils.toBytesArray(bytesArrayList);
+ assertEquals(bytesArray.length, 1);
+ assertEquals(bytesArray[0], new ByteArray(new byte[]{0x01, 0x02}));
+
+ bytesArrayList.add(new ByteArray(new byte[]{0x03}));
+ bytesArrayList.add(new ByteArray(new byte[]{0x04, 0x05, 0x06}));
+ bytesArray = ArrayListUtils.toBytesArray(bytesArrayList);
+ assertEquals(bytesArray.length, 3);
+ assertEquals(bytesArray[0], new ByteArray(new byte[]{0x01, 0x02}));
+ assertEquals(bytesArray[1], new ByteArray(new byte[]{0x03}));
+ assertEquals(bytesArray[2], new ByteArray(new byte[]{0x04, 0x05, 0x06}));
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/BlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/common/BlockValSet.java
index 495f517dfa8..dad7da5c3bf 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/BlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/BlockValSet.java
@@ -164,6 +164,13 @@ public interface BlockValSet {
*/
double[][] getDoubleValuesMV();
+ /**
+ * Returns the BigDecimal values for a multi-valued column.
+ *
+ * @return Array of BigDecimal values
+ */
+ BigDecimal[][] getBigDecimalValuesMV();
+
/**
* Returns the string values for a multi-valued column.
*
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/DataBlockCache.java b/pinot-core/src/main/java/org/apache/pinot/core/common/DataBlockCache.java
index 18ddda44b0c..e902f316ee7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/DataBlockCache.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/DataBlockCache.java
@@ -357,6 +357,24 @@ public class DataBlockCache implements AutoCloseable {
return doubleValues;
}
+ /**
+ * Get the BigDecimal values for a multi-valued column.
+ *
+ * @param column Column name
+ * @return Array of BigDecimal values
+ */
+ public BigDecimal[][] getBigDecimalValuesForMVColumn(String column) {
+ BigDecimal[][] bigDecimalValues = getValues(FieldSpec.DataType.BIG_DECIMAL, column);
+ if (markLoaded(FieldSpec.DataType.BIG_DECIMAL, column)) {
+ if (bigDecimalValues == null) {
+ bigDecimalValues = new BigDecimal[_length][];
+ putValues(FieldSpec.DataType.BIG_DECIMAL, column, bigDecimalValues);
+ }
+ _dataFetcher.fetchBigDecimalValues(column, _docIds, _length, bigDecimalValues);
+ }
+ return bigDecimalValues;
+ }
+
/**
* Get the string values for a multi-valued column.
*
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
index 617111938fc..16a74918c6b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
@@ -264,6 +264,18 @@ public class DataFetcher implements AutoCloseable {
_columnValueReaderMap.get(column).readDoubleValuesMV(inDocIds, length, outValues);
}
+ /**
+ * Fetch the BigDecimal values for a multi-valued column.
+ *
+ * @param column Column name
+ * @param inDocIds Input document Ids buffer
+ * @param length Number of input document Ids
+ * @param outValues Buffer for output
+ */
+ public void fetchBigDecimalValues(String column, int[] inDocIds, int length, BigDecimal[][] outValues) {
+ _columnValueReaderMap.get(column).readBigDecimalValuesMV(inDocIds, length, outValues);
+ }
+
/**
* Fetch the string values for a multi-valued column.
*
@@ -547,6 +559,21 @@ public class DataFetcher implements AutoCloseable {
}
}
+ void readBigDecimalValuesMV(int[] docIds, int length, BigDecimal[][] valuesBuffer) {
+ Tracing.activeRecording().setInputDataType(_storedType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
+ if (_dictionary != null) {
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext);
+ BigDecimal[] values = new BigDecimal[numValues];
+ _dictionary.readBigDecimalValues(_reusableMVDictIds, numValues, values);
+ valuesBuffer[i] = values;
+ }
+ } else {
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry, valuesBuffer, readerContext);
+ }
+ }
+
void readStringValuesMV(int[] docIds, int length, String[][] valuesBuffer) {
Tracing.activeRecording().setInputDataType(_storedType, _singleValue);
ForwardIndexReaderContext readerContext = getReaderContext();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
index 6652f381995..24314296796 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/ObjectSerDeUtils.java
@@ -165,7 +165,10 @@ public class ObjectSerDeUtils {
TupleIntSketchAccumulator(48),
CpcSketchAccumulator(49),
OrderedStringSet(50),
- FunnelStepEventAccumulator(51);
+ FunnelStepEventAccumulator(51),
+ BigDecimalArrayList(52),
+ BigDecimalSet(53),
+ BytesArrayList(54);
private final int _value;
@@ -205,6 +208,12 @@ public class ObjectSerDeUtils {
if (next instanceof String) {
return ObjectType.StringArrayList;
}
+ if (next instanceof BigDecimal) {
+ return ObjectType.BigDecimalArrayList;
+ }
+ if (next instanceof ByteArray) {
+ return ObjectType.BytesArrayList;
+ }
throw new IllegalArgumentException(
"Unsupported type of value: " + next.getClass().getSimpleName());
}
@@ -256,6 +265,8 @@ public class ObjectSerDeUtils {
ObjectSet objectSet = (ObjectSet) value;
if (objectSet.isEmpty() || objectSet.iterator().next() instanceof String) {
return ObjectType.StringSet;
+ } else if (objectSet.iterator().next() instanceof BigDecimal) {
+ return ObjectType.BigDecimalSet;
} else {
return ObjectType.BytesSet;
}
@@ -1363,6 +1374,131 @@ public class ObjectSerDeUtils {
}
};
+ public static final ObjectSerDe<ObjectArrayList<BigDecimal>> BIG_DECIMAL_ARRAY_LIST_SER_DE =
+ new ObjectSerDe<ObjectArrayList<BigDecimal>>() {
+ @Override
+ public byte[] serialize(ObjectArrayList<BigDecimal> list) {
+ int size = list.size();
+ // Header: size, then a length-prefix per element, followed by the serialized BigDecimal bytes.
+ long bufferSize = (1 + (long) size) * Integer.BYTES;
+ byte[][] valueBytesArray = new byte[size][];
+ for (int i = 0; i < size; i++) {
+ byte[] valueBytes = BigDecimalUtils.serialize(list.get(i));
+ bufferSize += valueBytes.length;
+ valueBytesArray[i] = valueBytes;
+ }
+ Preconditions.checkState(bufferSize <= Integer.MAX_VALUE, "Buffer size exceeds 2GB");
+ byte[] bytes = new byte[(int) bufferSize];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ byteBuffer.putInt(size);
+ for (byte[] valueBytes : valueBytesArray) {
+ byteBuffer.putInt(valueBytes.length);
+ byteBuffer.put(valueBytes);
+ }
+ return bytes;
+ }
+
+ @Override
+ public ObjectArrayList<BigDecimal> deserialize(byte[] bytes) {
+ return deserialize(ByteBuffer.wrap(bytes));
+ }
+
+ @Override
+ public ObjectArrayList<BigDecimal> deserialize(ByteBuffer byteBuffer) {
+ int size = byteBuffer.getInt();
+ ObjectArrayList<BigDecimal> list = new ObjectArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ int length = byteBuffer.getInt();
+ byte[] valueBytes = new byte[length];
+ byteBuffer.get(valueBytes);
+ list.add(BigDecimalUtils.deserialize(valueBytes));
+ }
+ return list;
+ }
+ };
+
+ public static final ObjectSerDe<Set<BigDecimal>> BIG_DECIMAL_SET_SER_DE = new ObjectSerDe<Set<BigDecimal>>() {
+ @Override
+ public byte[] serialize(Set<BigDecimal> set) {
+ int size = set.size();
+ long bufferSize = (1 + (long) size) * Integer.BYTES;
+ byte[][] valueBytesArray = new byte[size][];
+ int index = 0;
+ for (BigDecimal value : set) {
+ byte[] valueBytes = BigDecimalUtils.serialize(value);
+ bufferSize += valueBytes.length;
+ valueBytesArray[index++] = valueBytes;
+ }
+ Preconditions.checkState(bufferSize <= Integer.MAX_VALUE, "Buffer size exceeds 2GB");
+ byte[] bytes = new byte[(int) bufferSize];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ byteBuffer.putInt(size);
+ for (byte[] valueBytes : valueBytesArray) {
+ byteBuffer.putInt(valueBytes.length);
+ byteBuffer.put(valueBytes);
+ }
+ return bytes;
+ }
+
+ @Override
+ public ObjectOpenHashSet<BigDecimal> deserialize(byte[] bytes) {
+ return deserialize(ByteBuffer.wrap(bytes));
+ }
+
+ @Override
+ public ObjectOpenHashSet<BigDecimal> deserialize(ByteBuffer byteBuffer) {
+ int size = byteBuffer.getInt();
+ ObjectOpenHashSet<BigDecimal> set = new ObjectOpenHashSet<>(size);
+ for (int i = 0; i < size; i++) {
+ int length = byteBuffer.getInt();
+ byte[] valueBytes = new byte[length];
+ byteBuffer.get(valueBytes);
+ set.add(BigDecimalUtils.deserialize(valueBytes));
+ }
+ return set;
+ }
+ };
+
+ public static final ObjectSerDe<ObjectArrayList<ByteArray>> BYTES_ARRAY_LIST_SER_DE =
+ new ObjectSerDe<ObjectArrayList<ByteArray>>() {
+ @Override
+ public byte[] serialize(ObjectArrayList<ByteArray> list) {
+ int size = list.size();
+ long bufferSize = (1 + (long) size) * Integer.BYTES;
+ for (int i = 0; i < size; i++) {
+ bufferSize += list.get(i).length();
+ }
+ Preconditions.checkState(bufferSize <= Integer.MAX_VALUE, "Buffer size exceeds 2GB");
+ byte[] bytes = new byte[(int) bufferSize];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ byteBuffer.putInt(size);
+ for (int i = 0; i < size; i++) {
+ byte[] valueBytes = list.get(i).getBytes();
+ byteBuffer.putInt(valueBytes.length);
+ byteBuffer.put(valueBytes);
+ }
+ return bytes;
+ }
+
+ @Override
+ public ObjectArrayList<ByteArray> deserialize(byte[] bytes) {
+ return deserialize(ByteBuffer.wrap(bytes));
+ }
+
+ @Override
+ public ObjectArrayList<ByteArray> deserialize(ByteBuffer byteBuffer) {
+ int size = byteBuffer.getInt();
+ ObjectArrayList<ByteArray> list = new ObjectArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ int length = byteBuffer.getInt();
+ byte[] valueBytes = new byte[length];
+ byteBuffer.get(valueBytes);
+ list.add(new ByteArray(valueBytes));
+ }
+ return list;
+ }
+ };
+
public static final ObjectSerDe<Int2LongMap> INT_2_LONG_MAP_SER_DE = new ObjectSerDe<Int2LongMap>() {
@Override
@@ -1803,6 +1939,9 @@ public class ObjectSerDeUtils {
DATA_SKETCH_CPC_ACCUMULATOR_SER_DE,
ORDERED_STRING_SET_SER_DE,
FUNNEL_STEP_EVENT_ACCUMULATOR_SER_DE,
+ BIG_DECIMAL_ARRAY_LIST_SER_DE,
+ BIG_DECIMAL_SET_SER_DE,
+ BYTES_ARRAY_LIST_SER_DE,
};
//@formatter:on
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java b/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
index c2fefc1b17e..d2620973f87 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
@@ -86,8 +86,12 @@ public class RowBasedBlockValueFetcher {
return new FloatMultiValueFetcher(blockValSet.getFloatValuesMV());
case DOUBLE:
return new DoubleMultiValueFetcher(blockValSet.getDoubleValuesMV());
+ case BIG_DECIMAL:
+ return new BigDecimalMultiValueFetcher(blockValSet.getBigDecimalValuesMV());
case STRING:
return new StringMultiValueFetcher(blockValSet.getStringValuesMV());
+ case BYTES:
+ return new BytesMultiValueFetcher(blockValSet.getBytesValuesMV());
default:
throw new IllegalStateException("Unsupported value type: " + storedType + " for multi-value column");
}
@@ -242,6 +246,18 @@ public class RowBasedBlockValueFetcher {
}
}
+ private static class BigDecimalMultiValueFetcher implements ValueFetcher {
+ private final BigDecimal[][] _values;
+
+ BigDecimalMultiValueFetcher(BigDecimal[][] values) {
+ _values = values;
+ }
+
+ public BigDecimal[] getValue(int docId) {
+ return _values[docId];
+ }
+ }
+
private static class StringMultiValueFetcher implements ValueFetcher {
private final String[][] _values;
@@ -254,6 +270,24 @@ public class RowBasedBlockValueFetcher {
}
}
+ /// Wraps each MV element into a [ByteArray] to match the SV convention ([BytesValueFetcher]).
+ private static class BytesMultiValueFetcher implements ValueFetcher {
+ private final byte[][][] _values;
+
+ BytesMultiValueFetcher(byte[][][] values) {
+ _values = values;
+ }
+
+ public ByteArray[] getValue(int docId) {
+ byte[][] raw = _values[docId];
+ ByteArray[] wrapped = new ByteArray[raw.length];
+ for (int i = 0; i < raw.length; i++) {
+ wrapped[i] = new ByteArray(raw[i]);
+ }
+ return wrapped;
+ }
+ }
+
private static class UnknownValueFetcher implements ValueFetcher {
UnknownValueFetcher() {
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
index 1aaebb5625e..5451dd6de93 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
@@ -165,9 +165,15 @@ public class DataBlockBuilder {
case DOUBLE_ARRAY:
setColumn(fixedSize, varSize, (double[]) value);
break;
+ case BIG_DECIMAL_ARRAY:
+ setColumn(fixedSize, varSize, (BigDecimal[]) value);
+ break;
case STRING_ARRAY:
setColumn(fixedSize, varSize, (String[]) value, dictionary);
break;
+ case BYTES_ARRAY:
+ setColumn(fixedSize, varSize, (ByteArray[]) value);
+ break;
// Null
case UNKNOWN:
setNull(fixedSize, varSize);
@@ -337,9 +343,8 @@ public class DataBlockBuilder {
case STRING: {
ToIntFunction<String> didSupplier = k -> dictionary.size();
int nullPlaceHolder = dictionary.computeIfAbsent((String) storedType.getNullPlaceholder(), didSupplier);
-
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -355,7 +360,7 @@ public class DataBlockBuilder {
case BYTES: {
ByteArray nullPlaceholder = (ByteArray) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -370,7 +375,7 @@ public class DataBlockBuilder {
case MAP: {
Map nullPlaceholder = (Map) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -386,7 +391,7 @@ public class DataBlockBuilder {
case INT_ARRAY: {
int[] nullPlaceholder = (int[]) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -401,7 +406,7 @@ public class DataBlockBuilder {
case LONG_ARRAY: {
long[] nullPlaceholder = (long[]) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -416,7 +421,7 @@ public class DataBlockBuilder {
case FLOAT_ARRAY: {
float[] nullPlaceholder = (float[]) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -431,7 +436,7 @@ public class DataBlockBuilder {
case DOUBLE_ARRAY: {
double[] nullPlaceholder = (double[]) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -443,10 +448,25 @@ public class DataBlockBuilder {
});
break;
}
+ case BIG_DECIMAL_ARRAY: {
+ BigDecimal[] nullPlaceholder = (BigDecimal[]) storedType.getNullPlaceholder();
+ interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
+ for (int rowId = start; rowId < end; rowId++) {
+ Object value = column[rowId];
+ if (value == null) {
+ nullBitmap.add(rowId);
+ setColumn(fixedSize, varSize, nullPlaceholder);
+ } else {
+ setColumn(fixedSize, varSize, (BigDecimal[]) value);
+ }
+ }
+ });
+ break;
+ }
case STRING_ARRAY: {
String[] nullPlaceholder = (String[]) storedType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
nullBitmap.add(rowId);
@@ -458,11 +478,26 @@ public class DataBlockBuilder {
});
break;
}
+ case BYTES_ARRAY: {
+ ByteArray[] nullPlaceholder = (ByteArray[]) storedType.getNullPlaceholder();
+ interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
+ for (int rowId = start; rowId < end; rowId++) {
+ Object value = column[rowId];
+ if (value == null) {
+ nullBitmap.add(rowId);
+ setColumn(fixedSize, varSize, nullPlaceholder);
+ } else {
+ setColumn(fixedSize, varSize, (ByteArray[]) value);
+ }
+ }
+ });
+ break;
+ }
// Custom intermediate result for aggregation function
case OBJECT: {
assert aggFunction != null;
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
if (value == null) {
setNull(fixedSize, varSize);
@@ -476,7 +511,7 @@ public class DataBlockBuilder {
// Null
case UNKNOWN:
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
- for (int rowId = 0; rowId < numRows; rowId++) {
+ for (int rowId = start; rowId < end; rowId++) {
setNull(fixedSize, varSize);
}
});
@@ -627,6 +662,17 @@ public class DataBlockBuilder {
}
}
+ private static void setColumn(ByteBuffer fixedSize, PagedPinotOutputStream varSize, BigDecimal[] values)
+ throws IOException {
+ writeVarOffsetInFixed(fixedSize, varSize);
+ fixedSize.putInt(values.length);
+ for (BigDecimal value : values) {
+ byte[] bytes = BigDecimalUtils.serialize(value);
+ varSize.writeInt(bytes.length);
+ varSize.write(bytes);
+ }
+ }
+
private static void setColumn(ByteBuffer fixedSize, PagedPinotOutputStream varSize, String[] values,
Object2IntOpenHashMap<String> dictionary)
throws IOException {
@@ -638,6 +684,17 @@ public class DataBlockBuilder {
}
}
+ private static void setColumn(ByteBuffer fixedSize, PagedPinotOutputStream varSize, ByteArray[] values)
+ throws IOException {
+ writeVarOffsetInFixed(fixedSize, varSize);
+ fixedSize.putInt(values.length);
+ for (ByteArray value : values) {
+ byte[] bytes = value.getBytes();
+ varSize.writeInt(bytes.length);
+ varSize.write(bytes);
+ }
+ }
+
private static void setColumn(ByteBuffer fixedSize, PagedPinotOutputStream varSize,
AggregationFunction.SerializedIntermediateResult value)
throws IOException {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTableBuilder.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTableBuilder.java
index a351937d9b0..4170cc849d8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTableBuilder.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTableBuilder.java
@@ -30,6 +30,7 @@ import org.apache.pinot.common.datatable.DataTableUtils;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
import org.apache.pinot.spi.utils.BigDecimalUtils;
+import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.MapUtils;
@@ -156,6 +157,32 @@ public abstract class BaseDataTableBuilder implements DataTableBuilder {
}
}
+ @Override
+ public void setColumn(int colId, BigDecimal[] values)
+ throws IOException {
+ _currentRowDataByteBuffer.position(_columnOffsets[colId]);
+ _currentRowDataByteBuffer.putInt(_variableSizeDataByteArrayOutputStream.size());
+ _currentRowDataByteBuffer.putInt(values.length);
+ for (BigDecimal value : values) {
+ byte[] bytes = BigDecimalUtils.serialize(value);
+ _variableSizeDataOutputStream.writeInt(bytes.length);
+ _variableSizeDataByteArrayOutputStream.write(bytes);
+ }
+ }
+
+ @Override
+ public void setColumn(int colId, ByteArray[] values)
+ throws IOException {
+ _currentRowDataByteBuffer.position(_columnOffsets[colId]);
+ _currentRowDataByteBuffer.putInt(_variableSizeDataByteArrayOutputStream.size());
+ _currentRowDataByteBuffer.putInt(values.length);
+ for (ByteArray value : values) {
+ byte[] bytes = value.getBytes();
+ _variableSizeDataOutputStream.writeInt(bytes.length);
+ _variableSizeDataByteArrayOutputStream.write(bytes);
+ }
+ }
+
@Override
public void setColumn(int colId, AggregationFunction.SerializedIntermediateResult value)
throws IOException {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilder.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilder.java
index bed93863b5a..7ab1288fcff 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilder.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableBuilder.java
@@ -81,10 +81,14 @@ public interface DataTableBuilder {
void setColumn(int colId, double[] values)
throws IOException;
+ void setColumn(int colId, BigDecimal[] values)
+ throws IOException;
+
void setColumn(int colId, String[] values)
throws IOException;
- // TODO: Support MV BYTES
+ void setColumn(int colId, ByteArray[] values)
+ throws IOException;
void setColumn(int colId, AggregationFunction.SerializedIntermediateResult value)
throws IOException;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
index 668e396ae27..8f493869e3b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
@@ -194,18 +194,18 @@ public class AggregationResultsBlock extends BaseResultsBlock {
case LONG:
dataTableBuilder.setColumn(index, (long) result);
break;
- case DOUBLE:
- dataTableBuilder.setColumn(index, (double) result);
- break;
- case STRING:
- dataTableBuilder.setColumn(index, result.toString());
- break;
case FLOAT:
dataTableBuilder.setColumn(index, (float) result);
break;
+ case DOUBLE:
+ dataTableBuilder.setColumn(index, (double) result);
+ break;
case BIG_DECIMAL:
dataTableBuilder.setColumn(index, (BigDecimal) result);
break;
+ case STRING:
+ dataTableBuilder.setColumn(index, result.toString());
+ break;
case BYTES:
dataTableBuilder.setColumn(index, (ByteArray) result);
break;
@@ -252,9 +252,15 @@ public class AggregationResultsBlock extends BaseResultsBlock {
case DOUBLE_ARRAY:
dataTableBuilder.setColumn(index, ArrayListUtils.toDoubleArray((DoubleArrayList) result));
break;
+ case BIG_DECIMAL_ARRAY:
+ dataTableBuilder.setColumn(index, ArrayListUtils.toBigDecimalArray((ObjectArrayList<BigDecimal>) result));
+ break;
case STRING_ARRAY:
dataTableBuilder.setColumn(index, ArrayListUtils.toStringArray((ObjectArrayList<String>) result));
break;
+ case BYTES_ARRAY:
+ dataTableBuilder.setColumn(index, ArrayListUtils.toBytesArray((ObjectArrayList<ByteArray>) result));
+ break;
default:
throw new IllegalStateException("Illegal column data type in final result: " + columnDataType);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
index 74a2790bcfa..ea1e9bab31f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
@@ -328,6 +328,15 @@ public class GroupByResultsBlock extends BaseResultsBlock {
dataTableBuilder.setColumn(columnIndex, (double[]) value);
}
break;
+ case BIG_DECIMAL_ARRAY:
+ if (value instanceof ObjectArrayList) {
+ //noinspection unchecked
+ dataTableBuilder.setColumn(columnIndex,
+ ArrayListUtils.toBigDecimalArray((ObjectArrayList<BigDecimal>) value));
+ } else {
+ dataTableBuilder.setColumn(columnIndex, (BigDecimal[]) value);
+ }
+ break;
case STRING_ARRAY:
if (value instanceof ObjectArrayList) {
//noinspection unchecked
@@ -336,6 +345,14 @@ public class GroupByResultsBlock extends BaseResultsBlock {
dataTableBuilder.setColumn(columnIndex, (String[]) value);
}
break;
+ case BYTES_ARRAY:
+ if (value instanceof ObjectArrayList) {
+ //noinspection unchecked
+ dataTableBuilder.setColumn(columnIndex, ArrayListUtils.toBytesArray((ObjectArrayList<ByteArray>) value));
+ } else {
+ dataTableBuilder.setColumn(columnIndex, (ByteArray[]) value);
+ }
+ break;
default:
throw new IllegalStateException("Unsupported stored type: " + storedColumnDataType);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/MVScanDocIdIterator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/MVScanDocIdIterator.java
index 04a3e8cfe7c..c4cdbf3c776 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/MVScanDocIdIterator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/dociditerators/MVScanDocIdIterator.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.dociditerators;
+import java.math.BigDecimal;
import java.util.Map;
import java.util.OptionalInt;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
@@ -151,6 +152,8 @@ public final class MVScanDocIdIterator implements ScanBasedDocIdIterator {
return new FloatMatcher();
case DOUBLE:
return new DoubleMatcher();
+ case BIG_DECIMAL:
+ return new BigDecimalMatcher();
case STRING:
return new StringMatcher();
case BYTES:
@@ -230,6 +233,18 @@ public final class MVScanDocIdIterator implements ScanBasedDocIdIterator {
}
}
+ private class BigDecimalMatcher implements ValueMatcher {
+
+ private final BigDecimal[] _buffer = new BigDecimal[_maxNumValuesPerMVEntry];
+
+ @Override
+ public boolean doesValueMatch(int docId) {
+ int length = _reader.getBigDecimalMV(docId, _buffer, _readerContext);
+ _numEntriesScanned += length;
+ return _predicateEvaluator.applyMV(_buffer, length);
+ }
+ }
+
private class StringMatcher implements ValueMatcher {
private final String[] _buffer = new String[_maxNumValuesPerMVEntry];
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/DataBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/DataBlockValSet.java
index 4b5eaab2fe4..a8bc631b674 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/DataBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/DataBlockValSet.java
@@ -139,6 +139,11 @@ public class DataBlockValSet implements BlockValSet {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] getStringValuesMV() {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredDataBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredDataBlockValSet.java
index 020c1679b67..2114bb6f1e6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredDataBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredDataBlockValSet.java
@@ -169,6 +169,11 @@ public class FilteredDataBlockValSet implements BlockValSet {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] getStringValuesMV() {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredRowBasedBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredRowBasedBlockValSet.java
index f94d2a03b92..ba107a8b9fe 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredRowBasedBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/FilteredRowBasedBlockValSet.java
@@ -294,6 +294,11 @@ public class FilteredRowBasedBlockValSet implements BlockValSet {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] getStringValuesMV() {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/ProjectionBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/ProjectionBlockValSet.java
index 0dfd8b52d65..e0613a4b84f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/ProjectionBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/ProjectionBlockValSet.java
@@ -226,6 +226,14 @@ public class ProjectionBlockValSet implements BlockValSet {
}
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ try (InvocationScope scope = Tracing.getTracer().createScope(ProjectionBlockValSet.class)) {
+ recordReadValues(scope, DataType.BIG_DECIMAL, false);
+ return _dataBlockCache.getBigDecimalValuesForMVColumn(_column);
+ }
+ }
+
@Override
public String[][] getStringValuesMV() {
try (InvocationScope scope = Tracing.getTracer().createScope(ProjectionBlockValSet.class)) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/RowBasedBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/RowBasedBlockValSet.java
index c18cd3acb48..9d204d70062 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/RowBasedBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/RowBasedBlockValSet.java
@@ -562,6 +562,48 @@ public class RowBasedBlockValSet implements BlockValSet {
return values;
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ int numRows = _rows.size();
+ BigDecimal[][] values = new BigDecimal[numRows][];
+ if (numRows == 0) {
+ return values;
+ }
+ if (_dataType == DataType.UNKNOWN) {
+ Arrays.fill(values, new BigDecimal[0]);
+ return values;
+ }
+ for (int i = 0; i < numRows; i++) {
+ Object storedValue = _rows.get(i)[_colId];
+ if (storedValue instanceof int[]) {
+ int[] intArray = (int[]) storedValue;
+ values[i] = new BigDecimal[intArray.length];
+ ArrayCopyUtils.copy(intArray, values[i], intArray.length);
+ } else if (storedValue instanceof long[]) {
+ long[] longArray = (long[]) storedValue;
+ values[i] = new BigDecimal[longArray.length];
+ ArrayCopyUtils.copy(longArray, values[i], longArray.length);
+ } else if (storedValue instanceof float[]) {
+ float[] floatArray = (float[]) storedValue;
+ values[i] = new BigDecimal[floatArray.length];
+ ArrayCopyUtils.copy(floatArray, values[i], floatArray.length);
+ } else if (storedValue instanceof double[]) {
+ double[] doubleArray = (double[]) storedValue;
+ values[i] = new BigDecimal[doubleArray.length];
+ ArrayCopyUtils.copy(doubleArray, values[i], doubleArray.length);
+ } else if (storedValue instanceof BigDecimal[]) {
+ values[i] = (BigDecimal[]) storedValue;
+ } else if (storedValue instanceof String[]) {
+ String[] stringArray = (String[]) storedValue;
+ values[i] = new BigDecimal[stringArray.length];
+ ArrayCopyUtils.copy(stringArray, values[i], stringArray.length);
+ } else {
+ throw new IllegalStateException("Unsupported data type: " + storedValue.getClass().getName());
+ }
+ }
+ return values;
+ }
+
@Override
public String[][] getStringValuesMV() {
int numRows = _rows.size();
@@ -578,27 +620,23 @@ public class RowBasedBlockValSet implements BlockValSet {
if (storedValue instanceof int[]) {
int[] intArray = (int[]) storedValue;
values[i] = new String[intArray.length];
- for (int j = 0; j < intArray.length; j++) {
- values[i][j] = Integer.toString(intArray[j]);
- }
+ ArrayCopyUtils.copy(intArray, values[i], intArray.length);
} else if (storedValue instanceof long[]) {
long[] longArray = (long[]) storedValue;
values[i] = new String[longArray.length];
- for (int j = 0; j < longArray.length; j++) {
- values[i][j] = Long.toString(longArray[j]);
- }
+ ArrayCopyUtils.copy(longArray, values[i], longArray.length);
} else if (storedValue instanceof float[]) {
float[] floatArray = (float[]) storedValue;
values[i] = new String[floatArray.length];
- for (int j = 0; j < floatArray.length; j++) {
- values[i][j] = Float.toString(floatArray[j]);
- }
+ ArrayCopyUtils.copy(floatArray, values[i], floatArray.length);
} else if (storedValue instanceof double[]) {
double[] doubleArray = (double[]) storedValue;
values[i] = new String[doubleArray.length];
- for (int j = 0; j < doubleArray.length; j++) {
- values[i][j] = Double.toString(doubleArray[j]);
- }
+ ArrayCopyUtils.copy(doubleArray, values[i], doubleArray.length);
+ } else if (storedValue instanceof BigDecimal[]) {
+ BigDecimal[] bigDecimalArray = (BigDecimal[]) storedValue;
+ values[i] = new String[bigDecimalArray.length];
+ ArrayCopyUtils.copy(bigDecimalArray, values[i], bigDecimalArray.length);
} else if (storedValue instanceof String[]) {
values[i] = (String[]) storedValue;
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/TransformBlockValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/TransformBlockValSet.java
index 823dd05cbe3..b8d77415725 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/TransformBlockValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/TransformBlockValSet.java
@@ -216,6 +216,14 @@ public class TransformBlockValSet implements BlockValSet {
}
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ try (InvocationScope scope = Tracing.getTracer().createScope(TransformBlockValSet.class)) {
+ recordTransformValues(scope, DataType.BIG_DECIMAL, false);
+ return _transformFunction.transformToBigDecimalValuesMV(_valueBlock);
+ }
+ }
+
@Override
public String[][] getStringValuesMV() {
try (InvocationScope scope = Tracing.getTracer().createScope(TransformBlockValSet.class)) {
@@ -271,12 +279,24 @@ public class TransformBlockValSet implements BlockValSet {
_numMVEntries[i] = doubleValues[i].length;
}
return _numMVEntries;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValues = getBigDecimalValuesMV();
+ for (int i = 0; i < numDocs; i++) {
+ _numMVEntries[i] = bigDecimalValues[i].length;
+ }
+ return _numMVEntries;
case STRING:
String[][] stringValues = getStringValuesMV();
for (int i = 0; i < numDocs; i++) {
_numMVEntries[i] = stringValues[i].length;
}
return _numMVEntries;
+ case BYTES:
+ byte[][][] bytesValues = getBytesValuesMV();
+ for (int i = 0; i < numDocs; i++) {
+ _numMVEntries[i] = bytesValues[i].length;
+ }
+ return _numMVEntries;
default:
throw new IllegalStateException();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
index 18d15d36735..7117cae7347 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
@@ -123,6 +123,11 @@ public abstract class BaseDictionaryBasedPredicateEvaluator extends BasePredicat
throw new UnsupportedOperationException();
}
+ @Override
+ public final boolean applyMV(BigDecimal[] values, int length) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public final boolean applySV(String value) {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
index c4179605856..f295b0e116d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
@@ -171,6 +171,26 @@ public abstract class BaseRawValueBasedPredicateEvaluator extends BasePredicateE
throw new UnsupportedOperationException();
}
+ @SuppressWarnings("Duplicates")
+ @Override
+ public boolean applyMV(BigDecimal[] values, int length) {
+ if (isExclusive()) {
+ for (int i = 0; i < length; i++) {
+ if (!applySV(values[i])) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ for (int i = 0; i < length; i++) {
+ if (applySV(values[i])) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
@Override
public boolean applySV(String value) {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
index 09b420f48f0..880178933f3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
@@ -246,6 +246,15 @@ public interface PredicateEvaluator {
*/
boolean applySV(BigDecimal value);
+ /**
+ * Apply a multi-value entry to the predicate.
+ *
+ * @param values Array of raw values
+ * @param length Number of values in the entry
+ * @return Whether the entry matches the predicate
+ */
+ boolean applyMV(BigDecimal[] values, int length);
+
/**
* Apply a single-value entry to the predicate.
*
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/NonScanBasedAggregationOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/NonScanBasedAggregationOperator.java
index 783fd85129c..c6def8ba0b0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/NonScanBasedAggregationOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/NonScanBasedAggregationOperator.java
@@ -27,6 +27,7 @@ import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -274,6 +275,13 @@ public class NonScanBasedAggregationOperator extends BaseOperator<AggregationRes
doubleSet.add(dictionary.getDoubleValue(dictId));
}
return doubleSet;
+ case BIG_DECIMAL:
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet = new ObjectOpenHashSet<>(dictionarySize);
+ for (int dictId = 0; dictId < dictionarySize; dictId++) {
+ QueryThreadContext.checkTerminationAndSampleUsagePeriodically(dictId, EXPLAIN_NAME);
+ bigDecimalSet.add(dictionary.getBigDecimalValue(dictId));
+ }
+ return bigDecimalSet;
case STRING:
ObjectOpenHashSet<String> stringSet = new ObjectOpenHashSet<>(dictionarySize);
for (int dictId = 0; dictId < dictionarySize; dictId++) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
index 8b65c42f268..32cc3633d5f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.transform.function;
+import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.ColumnContext;
@@ -94,12 +95,24 @@ public class ArrayLengthTransformFunction extends BaseTransformFunction {
_intValuesSV[i] = doubleValuesMV[i].length;
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = _argument.transformToBigDecimalValuesMV(valueBlock);
+ for (int i = 0; i < length; i++) {
+ _intValuesSV[i] = bigDecimalValuesMV[i].length;
+ }
+ break;
case STRING:
String[][] stringValuesMV = _argument.transformToStringValuesMV(valueBlock);
for (int i = 0; i < length; i++) {
_intValuesSV[i] = stringValuesMV[i].length;
}
break;
+ case BYTES:
+ byte[][][] bytesValuesMV = _argument.transformToBytesValuesMV(valueBlock);
+ for (int i = 0; i < length; i++) {
+ _intValuesSV[i] = bytesValuesMV[i].length;
+ }
+ break;
default:
throw new IllegalStateException();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
index d14f26f1603..64878aa218e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
@@ -37,6 +37,7 @@ import org.roaringbitmap.RoaringBitmap;
* The <code>LiteralTransformFunction</code> class is a special transform function which is a wrapper on top of a
* LITERAL. The data type is inferred from the literal string.
*/
+// TODO: Support BIG_DECIMAL literal type, then implement transformToBigDecimalValuesMV.
public class ArrayLiteralTransformFunction implements TransformFunction {
public static final String FUNCTION_NAME = "arrayValueConstructor";
@@ -442,6 +443,11 @@ public class ArrayLiteralTransformFunction implements TransformFunction {
return doubleArrayResult;
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
int numDocs = valueBlock.getNumDocs();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
index e2223ec392e..3b82a98bd3b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
@@ -66,7 +66,6 @@ public abstract class BaseTransformFunction implements TransformFunction {
new TransformResultMetadata(DataType.FLOAT, false, false);
protected static final TransformResultMetadata DOUBLE_MV_NO_DICTIONARY_METADATA =
new TransformResultMetadata(DataType.DOUBLE, false, false);
- // TODO: Support MV BIG_DECIMAL
protected static final TransformResultMetadata BIG_DECIMAL_MV_NO_DICTIONARY_METADATA =
new TransformResultMetadata(DataType.BIG_DECIMAL, false, false);
protected static final TransformResultMetadata BOOLEAN_MV_NO_DICTIONARY_METADATA =
@@ -96,6 +95,7 @@ public abstract class BaseTransformFunction implements TransformFunction {
protected long[][] _longValuesMV;
protected float[][] _floatValuesMV;
protected double[][] _doubleValuesMV;
+ protected BigDecimal[][] _bigDecimalValuesMV;
protected String[][] _stringValuesMV;
protected byte[][][] _bytesValuesMV;
@@ -525,6 +525,10 @@ public abstract class BaseTransformFunction implements TransformFunction {
double[][] doubleValuesMV = transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copy(doubleValuesMV, _intValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _intValuesMV, length);
+ break;
case STRING:
String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
ArrayCopyUtils.copy(stringValuesMV, _intValuesMV, length);
@@ -577,6 +581,10 @@ public abstract class BaseTransformFunction implements TransformFunction {
double[][] doubleValuesMV = transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copy(doubleValuesMV, _longValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _longValuesMV, length);
+ break;
case STRING:
String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
ArrayCopyUtils.copy(stringValuesMV, _longValuesMV, length);
@@ -629,6 +637,10 @@ public abstract class BaseTransformFunction implements TransformFunction {
double[][] doubleValuesMV = transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copy(doubleValuesMV, _floatValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _floatValuesMV, length);
+ break;
case STRING:
String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
ArrayCopyUtils.copy(stringValuesMV, _floatValuesMV, length);
@@ -681,6 +693,10 @@ public abstract class BaseTransformFunction implements TransformFunction {
float[][] floatValuesMV = transformToFloatValuesMV(valueBlock);
ArrayCopyUtils.copy(floatValuesMV, _doubleValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _doubleValuesMV, length);
+ break;
case STRING:
String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
ArrayCopyUtils.copy(stringValuesMV, _doubleValuesMV, length);
@@ -698,6 +714,62 @@ public abstract class BaseTransformFunction implements TransformFunction {
return _doubleValuesMV;
}
+ protected void initBigDecimalValuesMV(int length) {
+ if (_bigDecimalValuesMV == null || _bigDecimalValuesMV.length < length) {
+ _bigDecimalValuesMV = new BigDecimal[length][];
+ }
+ }
+
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ int length = valueBlock.getNumDocs();
+ initBigDecimalValuesMV(length);
+ Dictionary dictionary = getDictionary();
+ if (dictionary != null) {
+ int[][] dictIdsMV = transformToDictIdsMV(valueBlock);
+ for (int i = 0; i < length; i++) {
+ int[] dictIds = dictIdsMV[i];
+ int numValues = dictIds.length;
+ BigDecimal[] bigDecimalValues = new BigDecimal[numValues];
+ dictionary.readBigDecimalValues(dictIds, numValues, bigDecimalValues);
+ _bigDecimalValuesMV[i] = bigDecimalValues;
+ }
+ } else {
+ DataType resultDataType = getResultMetadata().getDataType();
+ switch (resultDataType.getStoredType()) {
+ case INT:
+ int[][] intValuesMV = transformToIntValuesMV(valueBlock);
+ ArrayCopyUtils.copy(intValuesMV, _bigDecimalValuesMV, length);
+ break;
+ case LONG:
+ long[][] longValuesMV = transformToLongValuesMV(valueBlock);
+ ArrayCopyUtils.copy(longValuesMV, _bigDecimalValuesMV, length);
+ break;
+ case FLOAT:
+ float[][] floatValuesMV = transformToFloatValuesMV(valueBlock);
+ ArrayCopyUtils.copy(floatValuesMV, _bigDecimalValuesMV, length);
+ break;
+ case DOUBLE:
+ double[][] doubleValuesMV = transformToDoubleValuesMV(valueBlock);
+ ArrayCopyUtils.copy(doubleValuesMV, _bigDecimalValuesMV, length);
+ break;
+ case STRING:
+ String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
+ ArrayCopyUtils.copy(stringValuesMV, _bigDecimalValuesMV, length);
+ break;
+ case UNKNOWN:
+ // Copy the values to ensure behaviour consistency with non null-handling.
+ for (int i = 0; i < length; i++) {
+ _bigDecimalValuesMV[i] = NullValuePlaceHolder.BIG_DECIMAL_ARRAY;
+ }
+ break;
+ default:
+ throw new IllegalStateException(String.format("Cannot read MV %s as BIG_DECIMAL", resultDataType));
+ }
+ }
+ return _bigDecimalValuesMV;
+ }
+
protected void initStringValuesMV(int length) {
if (_stringValuesMV == null || _stringValuesMV.length < length) {
_stringValuesMV = new String[length][];
@@ -737,6 +809,10 @@ public abstract class BaseTransformFunction implements TransformFunction {
double[][] doubleValuesMV = transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copy(doubleValuesMV, _stringValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _stringValuesMV, length);
+ break;
case UNKNOWN:
// Copy the values to ensure behaviour consistency with non null-handling.
for (int i = 0; i < length; i++) {
@@ -771,9 +847,25 @@ public abstract class BaseTransformFunction implements TransformFunction {
_bytesValuesMV[i] = bytesValues;
}
} else {
- assert getResultMetadata().getDataType().getStoredType() == DataType.STRING;
- String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
- ArrayCopyUtils.copy(stringValuesMV, _bytesValuesMV, length);
+ DataType resultDataType = getResultMetadata().getDataType();
+ switch (resultDataType.getStoredType()) {
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _bytesValuesMV, length);
+ break;
+ case STRING:
+ String[][] stringValuesMV = transformToStringValuesMV(valueBlock);
+ ArrayCopyUtils.copy(stringValuesMV, _bytesValuesMV, length);
+ break;
+ case UNKNOWN:
+ // Copy the values to ensure behaviour consistency with non null-handling.
+ for (int i = 0; i < length; i++) {
+ _bytesValuesMV[i] = NullValuePlaceHolder.BYTES_ARRAY;
+ }
+ break;
+ default:
+ throw new IllegalStateException(String.format("Cannot read MV %s as BYTES", resultDataType));
+ }
}
return _bytesValuesMV;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
index 4e7de7fb951..6d61c1c99ef 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import com.google.common.base.Preconditions;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
@@ -77,9 +76,7 @@ public class CastTransformFunction extends BaseTransformFunction {
case "DECIMAL":
case "BIGDECIMAL":
case "BIG_DECIMAL":
- // TODO: Support MV BIG_DECIMAL
- Preconditions.checkState(sourceSV, "Cannot cast from MV to BIG_DECIMAL");
- _resultMetadata = BIG_DECIMAL_SV_NO_DICTIONARY_METADATA;
+ _resultMetadata = sourceSV ? BIG_DECIMAL_SV_NO_DICTIONARY_METADATA : BIG_DECIMAL_MV_NO_DICTIONARY_METADATA;
break;
case "BOOL":
case "BOOLEAN":
@@ -112,6 +109,11 @@ public class CastTransformFunction extends BaseTransformFunction {
case "DOUBLE_ARRAY":
_resultMetadata = DOUBLE_MV_NO_DICTIONARY_METADATA;
break;
+ case "DECIMAL_ARRAY":
+ case "BIGDECIMAL_ARRAY":
+ case "BIG_DECIMAL_ARRAY":
+ _resultMetadata = BIG_DECIMAL_MV_NO_DICTIONARY_METADATA;
+ break;
case "STRING_ARRAY":
case "VARCHAR_ARRAY":
_resultMetadata = STRING_MV_NO_DICTIONARY_METADATA;
@@ -338,6 +340,10 @@ public class CastTransformFunction extends BaseTransformFunction {
double[][] doubleValuesMV = _transformFunction.transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copyToBoolean(doubleValuesMV, _intValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = _transformFunction.transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copyToBoolean(bigDecimalValuesMV, _intValuesMV, length);
+ break;
case STRING:
String[][] stringValuesMV = _transformFunction.transformToStringValuesMV(valueBlock);
ArrayCopyUtils.copyToBoolean(stringValuesMV, _intValuesMV, length);
@@ -391,6 +397,15 @@ public class CastTransformFunction extends BaseTransformFunction {
}
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.BIG_DECIMAL) {
+ return _transformFunction.transformToBigDecimalValuesMV(valueBlock);
+ } else {
+ return super.transformToBigDecimalValuesMV(valueBlock);
+ }
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
DataType resultDataType = _resultMetadata.getDataType();
@@ -431,6 +446,10 @@ public class CastTransformFunction extends BaseTransformFunction {
double[][] doubleValuesMV = _transformFunction.transformToDoubleValuesMV(valueBlock);
ArrayCopyUtils.copy(doubleValuesMV, _stringValuesMV, length);
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValuesMV = _transformFunction.transformToBigDecimalValuesMV(valueBlock);
+ ArrayCopyUtils.copy(bigDecimalValuesMV, _stringValuesMV, length);
+ break;
case BOOLEAN:
intValuesMV = transformToBooleanValuesMV(valueBlock);
ArrayCopyUtils.copyFromBoolean(intValuesMV, _stringValuesMV, length);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenerateArrayTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenerateArrayTransformFunction.java
index 09639174953..df7e42a1b15 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenerateArrayTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/GenerateArrayTransformFunction.java
@@ -390,6 +390,11 @@ public class GenerateArrayTransformFunction implements TransformFunction {
return doubleArrayResult;
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunction.java
index 1773af53cdb..48700c7cb0c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/IdentifierTransformFunction.java
@@ -135,6 +135,11 @@ public class IdentifierTransformFunction implements TransformFunction {
return valueBlock.getBlockValueSet(_columnName).getDoubleValuesMV();
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ return valueBlock.getBlockValueSet(_columnName).getBigDecimalValuesMV();
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
return valueBlock.getBlockValueSet(_columnName).getStringValuesMV();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunction.java
index 2161175b33b..9700a09b170 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LiteralTransformFunction.java
@@ -241,6 +241,11 @@ public class LiteralTransformFunction implements TransformFunction {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
index fe0adc86256..3d1abda8bff 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
@@ -436,9 +436,15 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
case PRIMITIVE_DOUBLE_ARRAY:
_nonLiteralValues[i] = transformFunction.transformToDoubleValuesMV(valueBlock);
break;
+ case BIG_DECIMAL_ARRAY:
+ _nonLiteralValues[i] = transformFunction.transformToBigDecimalValuesMV(valueBlock);
+ break;
case STRING_ARRAY:
_nonLiteralValues[i] = transformFunction.transformToStringValuesMV(valueBlock);
break;
+ case BYTES_ARRAY:
+ _nonLiteralValues[i] = transformFunction.transformToBytesValuesMV(valueBlock);
+ break;
default:
throw new IllegalStateException("Unsupported parameter type: " + parameterType);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
index 95ee40f7279..3a0a5720f2b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunction.java
@@ -152,6 +152,11 @@ public interface TransformFunction {
*/
double[][] transformToDoubleValuesMV(ValueBlock valueBlock);
+ /**
+ * Transforms the data from the given value block to multi-valued BigDecimal values.
+ */
+ BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock);
+
/**
* Transforms the data from the given value block to multi-valued string values.
*/
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
index 16fa17ab840..f17c85c0d26 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionFactory.java
@@ -23,6 +23,10 @@ import java.util.List;
import org.apache.datasketches.tuple.aninteger.IntegerSummary;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FunctionContext;
+import org.apache.pinot.core.query.aggregation.function.array.ArrayAggBigDecimalFunction;
+import org.apache.pinot.core.query.aggregation.function.array.ArrayAggBytesFunction;
+import org.apache.pinot.core.query.aggregation.function.array.ArrayAggDistinctBigDecimalFunction;
+import org.apache.pinot.core.query.aggregation.function.array.ArrayAggDistinctBytesFunction;
import org.apache.pinot.core.query.aggregation.function.array.ArrayAggDistinctDoubleFunction;
import org.apache.pinot.core.query.aggregation.function.array.ArrayAggDistinctFloatFunction;
import org.apache.pinot.core.query.aggregation.function.array.ArrayAggDistinctIntFunction;
@@ -323,8 +327,12 @@ public class AggregationFunctionFactory {
return new ArrayAggDistinctFloatFunction(firstArgument, nullHandlingEnabled);
case DOUBLE:
return new ArrayAggDistinctDoubleFunction(firstArgument, nullHandlingEnabled);
+ case BIG_DECIMAL:
+ return new ArrayAggDistinctBigDecimalFunction(firstArgument, nullHandlingEnabled);
case STRING:
return new ArrayAggDistinctStringFunction(firstArgument, nullHandlingEnabled);
+ case BYTES:
+ return new ArrayAggDistinctBytesFunction(firstArgument, nullHandlingEnabled);
default:
throw new IllegalArgumentException("Unsupported data type for ARRAY_AGG: " + dataType);
}
@@ -340,8 +348,12 @@ public class AggregationFunctionFactory {
return new ArrayAggFloatFunction(firstArgument, nullHandlingEnabled);
case DOUBLE:
return new ArrayAggDoubleFunction(firstArgument, nullHandlingEnabled);
+ case BIG_DECIMAL:
+ return new ArrayAggBigDecimalFunction(firstArgument, nullHandlingEnabled);
case STRING:
return new ArrayAggStringFunction(firstArgument, nullHandlingEnabled);
+ case BYTES:
+ return new ArrayAggBytesFunction(firstArgument, nullHandlingEnabled);
default:
throw new IllegalArgumentException("Unsupported data type for ARRAY_AGG: " + dataType);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
index 79913301a3d..010510d485e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
@@ -195,8 +195,12 @@ public class AggregationFunctionUtils {
return FloatArrayList.wrap(dataTable.getFloatArray(rowId, colId));
case DOUBLE_ARRAY:
return DoubleArrayList.wrap(dataTable.getDoubleArray(rowId, colId));
+ case BIG_DECIMAL_ARRAY:
+ return ObjectArrayList.wrap(dataTable.getBigDecimalArray(rowId, colId));
case STRING_ARRAY:
return ObjectArrayList.wrap(dataTable.getStringArray(rowId, colId));
+ case BYTES_ARRAY:
+ return ObjectArrayList.wrap(dataTable.getBytesArray(rowId, colId));
default:
throw new IllegalStateException("Illegal column data type in final result: " + columnDataType);
}
@@ -236,6 +240,8 @@ public class AggregationFunctionUtils {
return dataTable.getFloatArray(rowId, colId);
case DOUBLE_ARRAY:
return dataTable.getDoubleArray(rowId, colId);
+ case BIG_DECIMAL_ARRAY:
+ return dataTable.getBigDecimalArray(rowId, colId);
case BOOLEAN_ARRAY: {
int[] intValues = dataTable.getIntArray(rowId, colId);
int numValues = intValues.length;
@@ -256,6 +262,8 @@ public class AggregationFunctionUtils {
}
case STRING_ARRAY:
return dataTable.getStringArray(rowId, colId);
+ case BYTES_ARRAY:
+ return dataTable.getBytesArray(rowId, colId);
default:
throw new IllegalStateException("Illegal column data type in final result: " + columnDataType);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
index 2ba1d852892..25a189c77d7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/BaseDistinctAggregateAggregationFunction.java
@@ -27,6 +27,7 @@ import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongSet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import java.math.BigDecimal;
import java.util.Set;
import org.apache.pinot.common.CustomObject;
import org.apache.pinot.common.request.context.ExpressionContext;
@@ -160,6 +161,10 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
ObjectSerDeUtils.DOUBLE_SET_SER_DE.serialize((DoubleSet) set));
}
Object sampleValue = set.iterator().next();
+ if (sampleValue instanceof BigDecimal) {
+ return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.BigDecimalSet.getValue(),
+ ObjectSerDeUtils.BIG_DECIMAL_SET_SER_DE.serialize((Set<BigDecimal>) set));
+ }
if (sampleValue instanceof String) {
return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.StringSet.getValue(),
ObjectSerDeUtils.STRING_SET_SER_DE.serialize((Set<String>) set));
@@ -246,6 +251,16 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet = (ObjectOpenHashSet<BigDecimal>) valueSet;
+ BigDecimal[] bigDecimalValues = blockValSet.getBigDecimalValuesSV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ bigDecimalSet.add(bigDecimalValues[i]);
+ }
+ });
+ break;
case STRING:
ObjectOpenHashSet<String> stringSet = (ObjectOpenHashSet<String>) valueSet;
String[] stringValues = blockValSet.getStringValuesSV();
@@ -342,6 +357,18 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet = (ObjectOpenHashSet<BigDecimal>) valueSet;
+ BigDecimal[][] bigDecimalValues = blockValSet.getBigDecimalValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ for (BigDecimal value : bigDecimalValues[i]) {
+ bigDecimalSet.add(value);
+ }
+ }
+ });
+ break;
case STRING:
ObjectOpenHashSet<String> stringSet = (ObjectOpenHashSet<String>) valueSet;
String[][] stringValues = blockValSet.getStringValuesMV();
@@ -356,6 +383,18 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BYTES:
+ ObjectOpenHashSet<ByteArray> bytesSet = (ObjectOpenHashSet<ByteArray>) valueSet;
+ byte[][][] bytesValues = blockValSet.getBytesValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ for (byte[] value : bytesValues[i]) {
+ bytesSet.add(new ByteArray(value));
+ }
+ }
+ });
+ break;
default:
throw new IllegalStateException(
"Illegal data type for " + _functionType.getName() + " aggregation function: " + storedType);
@@ -420,6 +459,16 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValues = blockValSet.getBigDecimalValuesSV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ ((ObjectOpenHashSet<BigDecimal>) getValueSet(groupByResultHolder, groupKeyArray[i], DataType.BIG_DECIMAL))
+ .add(bigDecimalValues[i]);
+ }
+ });
+ break;
case STRING:
String[] stringValues = blockValSet.getStringValuesSV();
@@ -517,6 +566,19 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValues = blockValSet.getBigDecimalValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet = (ObjectOpenHashSet<BigDecimal>) getValueSet(
+ groupByResultHolder, groupKeyArray[i], DataType.BIG_DECIMAL);
+ for (BigDecimal value : bigDecimalValues[i]) {
+ bigDecimalSet.add(value);
+ }
+ }
+ });
+ break;
case STRING:
String[][] stringValues = blockValSet.getStringValuesMV();
@@ -532,6 +594,19 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BYTES:
+ byte[][][] bytesValues = blockValSet.getBytesValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ ObjectOpenHashSet<ByteArray> bytesSet =
+ (ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder, groupKeyArray[i], DataType.BYTES);
+ for (byte[] value : bytesValues[i]) {
+ bytesSet.add(new ByteArray(value));
+ }
+ }
+ });
+ break;
default:
throw new IllegalStateException(
"Illegal data type for " + _functionType.getName() + " aggregation function: " + storedType);
@@ -595,6 +670,15 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValues = blockValSet.getBigDecimalValuesSV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ setValueForGroupKeys(groupByResultHolder, groupKeysArray[i], bigDecimalValues[i]);
+ }
+ });
+ break;
case STRING:
String[] stringValues = blockValSet.getStringValuesSV();
@@ -699,6 +783,21 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BIG_DECIMAL:
+ BigDecimal[][] bigDecimalValues = blockValSet.getBigDecimalValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ for (int groupKey : groupKeysArray[i]) {
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet =
+ (ObjectOpenHashSet<BigDecimal>) getValueSet(groupByResultHolder, groupKey, DataType.BIG_DECIMAL);
+ for (BigDecimal value : bigDecimalValues[i]) {
+ bigDecimalSet.add(value);
+ }
+ }
+ }
+ });
+ break;
case STRING:
String[][] stringValues = blockValSet.getStringValuesMV();
@@ -716,6 +815,21 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
});
break;
+ case BYTES:
+ byte[][][] bytesValues = blockValSet.getBytesValuesMV();
+
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ for (int groupKey : groupKeysArray[i]) {
+ ObjectOpenHashSet<ByteArray> bytesSet =
+ (ObjectOpenHashSet<ByteArray>) getValueSet(groupByResultHolder, groupKey, DataType.BYTES);
+ for (byte[] value : bytesValues[i]) {
+ bytesSet.add(new ByteArray(value));
+ }
+ }
+ }
+ });
+ break;
default:
throw new IllegalStateException(
"Illegal data type for " + _functionType.getName() + " aggregation function: " + storedType);
@@ -747,6 +861,7 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
return new FloatOpenHashSet();
case DOUBLE:
return new DoubleOpenHashSet();
+ case BIG_DECIMAL:
case STRING:
case BYTES:
return new ObjectOpenHashSet();
@@ -826,6 +941,15 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
}
}
+ /**
+ * Helper method to set BIG_DECIMAL value for the given group keys into the result holder.
+ */
+ private static void setValueForGroupKeys(GroupByResultHolder groupByResultHolder, int[] groupKeys, BigDecimal value) {
+ for (int groupKey : groupKeys) {
+ ((ObjectOpenHashSet<BigDecimal>) getValueSet(groupByResultHolder, groupKey, DataType.BIG_DECIMAL)).add(value);
+ }
+ }
+
/**
* Helper method to set STRING value for the given group keys into the result holder.
*/
@@ -878,6 +1002,12 @@ public abstract class BaseDistinctAggregateAggregationFunction<T extends Compara
doubleSet.add(dictionary.getDoubleValue(iterator.next()));
}
return doubleSet;
+ case BIG_DECIMAL:
+ ObjectOpenHashSet<BigDecimal> bigDecimalSet = new ObjectOpenHashSet<>(numValues);
+ while (iterator.hasNext()) {
+ bigDecimalSet.add(dictionary.getBigDecimalValue(iterator.next()));
+ }
+ return bigDecimalSet;
case STRING:
ObjectOpenHashSet<String> stringSet = new ObjectOpenHashSet<>(numValues);
while (iterator.hasNext()) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/ParentExprMinMaxAggregationFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/ParentExprMinMaxAggregationFunction.java
index 15daa2f2a47..e094be206e7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/ParentExprMinMaxAggregationFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/ParentExprMinMaxAggregationFunction.java
@@ -18,13 +18,13 @@
*/
package org.apache.pinot.core.query.aggregation.function;
-import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.pinot.common.CustomObject;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.common.ObjectSerDeUtils;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
@@ -106,7 +106,6 @@ public class ParentExprMinMaxAggregationFunction extends ParentAggregationFuncti
return new ObjectGroupByResultHolder(initialCapacity, maxCapacity);
}
- @SuppressWarnings("LoopStatementThatDoesntLoop")
@Override
public void aggregate(int length, AggregationResultHolder aggregationResultHolder,
Map<ExpressionContext, BlockValSet> blockValSetMap) {
@@ -169,116 +168,16 @@ public class ParentExprMinMaxAggregationFunction extends ParentAggregationFuncti
List<ExprMinMaxProjectionValSetWrapper> exprMinMaxWrapperProjectionColumnSets =
_exprMinMaxWrapperProjectionColumnSets.get();
String[] projectionColNames = new String[_projectionColumns.size()];
- DataSchema.ColumnDataType[] projectionColTypes = new DataSchema.ColumnDataType[_projectionColumns.size()];
+ ColumnDataType[] projectionColTypes = new ColumnDataType[_projectionColumns.size()];
for (int i = 0; i < _projectionColumns.size(); i++) {
projectionColNames[i] = _projectionColumns.get(i).toString();
- ExpressionContext projectionColumn = _projectionColumns.get(i);
- BlockValSet blockValSet = blockValSetMap.get(projectionColumn);
- if (blockValSet.isSingleValue()) {
- switch (blockValSet.getValueType()) {
- case INT:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.INT, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.INT;
- break;
- case BOOLEAN:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.BOOLEAN, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.INT;
- break;
- case LONG:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.LONG, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.LONG;
- break;
- case TIMESTAMP:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.TIMESTAMP, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.LONG;
- break;
- case FLOAT:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.FLOAT, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.FLOAT;
- break;
- case DOUBLE:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.DOUBLE, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.DOUBLE;
- break;
- case STRING:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.STRING, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.STRING;
- break;
- case JSON:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.JSON, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.STRING;
- break;
- case BYTES:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.BYTES, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.BYTES;
- break;
- case BIG_DECIMAL:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(true, DataSchema.ColumnDataType.BIG_DECIMAL, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.BIG_DECIMAL;
- break;
- default:
- throw new IllegalStateException(
- "Cannot compute exprminMax projection on non-comparable type: " + blockValSet.getValueType());
- }
- } else {
- switch (blockValSet.getValueType()) {
- case INT:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.INT_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.INT_ARRAY;
- break;
- case BOOLEAN:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.BOOLEAN_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.INT_ARRAY;
- break;
- case LONG:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.LONG_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.LONG_ARRAY;
- break;
- case TIMESTAMP:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.TIMESTAMP_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.LONG_ARRAY;
- break;
- case FLOAT:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.FLOAT_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.FLOAT_ARRAY;
- break;
- case DOUBLE:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.DOUBLE_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.DOUBLE_ARRAY;
- break;
- case STRING:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.STRING_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.STRING_ARRAY;
- break;
- case BYTES:
- exprMinMaxWrapperProjectionColumnSets.add(
- new ExprMinMaxProjectionValSetWrapper(false, DataSchema.ColumnDataType.BYTES_ARRAY, blockValSet));
- projectionColTypes[i] = DataSchema.ColumnDataType.BYTES_ARRAY;
- break;
- default:
- throw new IllegalStateException(
- "Cannot compute exprminMax projection on non-comparable type: " + blockValSet.getValueType());
- }
- }
+ BlockValSet blockValSet = blockValSetMap.get(_projectionColumns.get(i));
+ ExprMinMaxProjectionValSetWrapper wrapper = new ExprMinMaxProjectionValSetWrapper(blockValSet);
+ exprMinMaxWrapperProjectionColumnSets.add(wrapper);
+ // TODO: Revisit if we should put actual type instead of stored type
+ projectionColTypes[i] = wrapper.getStoredType();
}
- // setup measuring column schema
+ // setup projection column schema
_projectionColumnSchema.set(new DataSchema(projectionColNames, projectionColTypes));
}
@@ -286,58 +185,14 @@ public class ParentExprMinMaxAggregationFunction extends ParentAggregationFuncti
List<ExprMinMaxMeasuringValSetWrapper> exprMinMaxWrapperMeasuringColumnSets =
_exprMinMaxWrapperMeasuringColumnSets.get();
String[] measuringColNames = new String[_numMeasuringColumns];
- DataSchema.ColumnDataType[] measuringColTypes = new DataSchema.ColumnDataType[_numMeasuringColumns];
+ ColumnDataType[] measuringColTypes = new ColumnDataType[_numMeasuringColumns];
for (int i = 0; i < _numMeasuringColumns; i++) {
measuringColNames[i] = _measuringColumns.get(i).toString();
- ExpressionContext measuringColumn = _measuringColumns.get(i);
- BlockValSet blockValSet = blockValSetMap.get(measuringColumn);
- Preconditions.checkState(blockValSet.isSingleValue(), "ExprMinMax only supports single-valued"
- + " measuring columns");
- switch (blockValSet.getValueType()) {
- case INT:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.INT, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.INT;
- break;
- case BOOLEAN:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.BOOLEAN, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.INT;
- break;
- case LONG:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.LONG, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.LONG;
- break;
- case TIMESTAMP:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.TIMESTAMP, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.LONG;
- break;
- case FLOAT:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.FLOAT, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.FLOAT;
- break;
- case DOUBLE:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.DOUBLE, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.DOUBLE;
- break;
- case STRING:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.STRING, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.STRING;
- break;
- case BIG_DECIMAL:
- exprMinMaxWrapperMeasuringColumnSets.add(
- new ExprMinMaxMeasuringValSetWrapper(true, DataSchema.ColumnDataType.BIG_DECIMAL, blockValSet));
- measuringColTypes[i] = DataSchema.ColumnDataType.BIG_DECIMAL;
- break;
- default:
- throw new IllegalStateException(
- "Cannot compute exprminMax measuring on non-comparable type: " + blockValSet.getValueType());
- }
+ BlockValSet blockValSet = blockValSetMap.get(_measuringColumns.get(i));
+ ExprMinMaxMeasuringValSetWrapper wrapper = new ExprMinMaxMeasuringValSetWrapper(blockValSet);
+ exprMinMaxWrapperMeasuringColumnSets.add(wrapper);
+ // TODO: Revisit if we should put actual type instead of stored type
+ measuringColTypes[i] = wrapper.getStoredType();
}
// setup measuring column schema
_measuringColumnSchema.set(new DataSchema(measuringColNames, measuringColTypes));
@@ -350,17 +205,16 @@ public class ParentExprMinMaxAggregationFunction extends ParentAggregationFuncti
}
_schemaInitialized.set(true);
String[] measuringColNames = new String[_numMeasuringColumns];
- DataSchema.ColumnDataType[] measuringColTypes = new DataSchema.ColumnDataType[_numMeasuringColumns];
+ ColumnDataType[] measuringColTypes = new ColumnDataType[_numMeasuringColumns];
for (int i = 0; i < _numMeasuringColumns; i++) {
measuringColNames[i] = _measuringColumns.get(i).toString();
- measuringColTypes[i] = DataSchema.ColumnDataType.STRING;
+ measuringColTypes[i] = ColumnDataType.STRING;
}
-
String[] projectionColNames = new String[_numProjectionColumns];
- DataSchema.ColumnDataType[] projectionColTypes = new DataSchema.ColumnDataType[_numProjectionColumns];
+ ColumnDataType[] projectionColTypes = new ColumnDataType[_numProjectionColumns];
for (int i = 0; i < _numProjectionColumns; i++) {
projectionColNames[i] = _projectionColumns.get(i).toString();
- projectionColTypes[i] = DataSchema.ColumnDataType.STRING;
+ projectionColTypes[i] = ColumnDataType.STRING;
}
_measuringColumnSchema.set(new DataSchema(measuringColNames, measuringColTypes));
_projectionColumnSchema.set(new DataSchema(projectionColNames, projectionColTypes));
@@ -423,8 +277,8 @@ public class ParentExprMinMaxAggregationFunction extends ParentAggregationFuncti
}
@Override
- public DataSchema.ColumnDataType getIntermediateResultColumnType() {
- return DataSchema.ColumnDataType.OBJECT;
+ public ColumnDataType getIntermediateResultColumnType() {
+ return ColumnDataType.OBJECT;
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBigDecimalFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBigDecimalFunction.java
new file mode 100644
index 00000000000..00e9dea9326
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBigDecimalFunction.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import java.math.BigDecimal;
+import java.util.Map;
+import org.apache.pinot.common.CustomObject;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.common.ObjectSerDeUtils;
+import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+
+
+public class ArrayAggBigDecimalFunction extends BaseArrayAggBigDecimalFunction<ObjectArrayList<BigDecimal>> {
+ public ArrayAggBigDecimalFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, nullHandlingEnabled);
+ }
+
+ @Override
+ public void aggregate(int length, AggregationResultHolder aggregationResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ ObjectArrayList<BigDecimal> valueArray =
+ aggregationResultHolder.getResult() != null ? aggregationResultHolder.getResult()
+ : new ObjectArrayList<>(length);
+ if (blockValSet.isSingleValue()) {
+ BigDecimal[] values = blockValSet.getBigDecimalValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ valueArray.add(values[i]);
+ }
+ });
+ } else {
+ BigDecimal[][] valuesArray = blockValSet.getBigDecimalValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ BigDecimal[] values = valuesArray[i];
+ for (BigDecimal v : values) {
+ valueArray.add(v);
+ }
+ }
+ });
+ }
+ aggregationResultHolder.setValue(valueArray);
+ }
+
+ @Override
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, BigDecimal value) {
+ ObjectArrayList<BigDecimal> valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new ObjectArrayList<>();
+ resultHolder.setValueForKey(groupKey, valueArray);
+ }
+ valueArray.add(value);
+ }
+
+ @Override
+ public SerializedIntermediateResult serializeIntermediateResult(ObjectArrayList<BigDecimal> list) {
+ return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.BigDecimalArrayList.getValue(),
+ ObjectSerDeUtils.BIG_DECIMAL_ARRAY_LIST_SER_DE.serialize(list));
+ }
+
+ @Override
+ public ObjectArrayList<BigDecimal> deserializeIntermediateResult(CustomObject customObject) {
+ return ObjectSerDeUtils.BIG_DECIMAL_ARRAY_LIST_SER_DE.deserialize(customObject.getBuffer());
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBytesFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBytesFunction.java
new file mode 100644
index 00000000000..df796352abd
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggBytesFunction.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import java.util.Map;
+import org.apache.pinot.common.CustomObject;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.common.ObjectSerDeUtils;
+import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+import org.apache.pinot.spi.utils.ByteArray;
+
+
+public class ArrayAggBytesFunction extends BaseArrayAggBytesFunction<ObjectArrayList<ByteArray>> {
+ public ArrayAggBytesFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, nullHandlingEnabled);
+ }
+
+ @Override
+ public void aggregate(int length, AggregationResultHolder aggregationResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ ObjectArrayList<ByteArray> valueArray =
+ aggregationResultHolder.getResult() != null ? aggregationResultHolder.getResult()
+ : new ObjectArrayList<>(length);
+ if (blockValSet.isSingleValue()) {
+ byte[][] values = blockValSet.getBytesValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ valueArray.add(new ByteArray(values[i]));
+ }
+ });
+ } else {
+ byte[][][] valuesArray = blockValSet.getBytesValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ byte[][] values = valuesArray[i];
+ for (byte[] v : values) {
+ valueArray.add(new ByteArray(v));
+ }
+ }
+ });
+ }
+ aggregationResultHolder.setValue(valueArray);
+ }
+
+ @Override
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, byte[] value) {
+ ObjectArrayList<ByteArray> valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new ObjectArrayList<>();
+ resultHolder.setValueForKey(groupKey, valueArray);
+ }
+ valueArray.add(new ByteArray(value));
+ }
+
+ @Override
+ public ObjectArrayList<ByteArray> extractFinalResult(ObjectArrayList<ByteArray> intermediateResult) {
+ return intermediateResult == null ? new ObjectArrayList<>() : intermediateResult;
+ }
+
+ @Override
+ public SerializedIntermediateResult serializeIntermediateResult(ObjectArrayList<ByteArray> list) {
+ return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.BytesArrayList.getValue(),
+ ObjectSerDeUtils.BYTES_ARRAY_LIST_SER_DE.serialize(list));
+ }
+
+ @Override
+ public ObjectArrayList<ByteArray> deserializeIntermediateResult(CustomObject customObject) {
+ return ObjectSerDeUtils.BYTES_ARRAY_LIST_SER_DE.deserialize(customObject.getBuffer());
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBigDecimalFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBigDecimalFunction.java
new file mode 100644
index 00000000000..da991783094
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBigDecimalFunction.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import it.unimi.dsi.fastutil.objects.ObjectSet;
+import java.math.BigDecimal;
+import java.util.Map;
+import org.apache.pinot.common.CustomObject;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.common.ObjectSerDeUtils;
+import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+
+
+public class ArrayAggDistinctBigDecimalFunction extends BaseArrayAggBigDecimalFunction<ObjectSet<BigDecimal>> {
+ public ArrayAggDistinctBigDecimalFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, nullHandlingEnabled);
+ }
+
+ @Override
+ public void aggregate(int length, AggregationResultHolder aggregationResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ ObjectOpenHashSet<BigDecimal> valueSet =
+ aggregationResultHolder.getResult() != null ? aggregationResultHolder.getResult()
+ : new ObjectOpenHashSet<>(length);
+ if (blockValSet.isSingleValue()) {
+ BigDecimal[] values = blockValSet.getBigDecimalValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ valueSet.add(values[i]);
+ }
+ });
+ } else {
+ BigDecimal[][] valuesArray = blockValSet.getBigDecimalValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ BigDecimal[] values = valuesArray[i];
+ for (BigDecimal v : values) {
+ valueSet.add(v);
+ }
+ }
+ });
+ }
+ aggregationResultHolder.setValue(valueSet);
+ }
+
+ @Override
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, BigDecimal value) {
+ ObjectOpenHashSet<BigDecimal> valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new ObjectOpenHashSet<>();
+ resultHolder.setValueForKey(groupKey, valueSet);
+ }
+ valueSet.add(value);
+ }
+
+ @Override
+ public SerializedIntermediateResult serializeIntermediateResult(ObjectSet<BigDecimal> set) {
+ return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.BigDecimalSet.getValue(),
+ ObjectSerDeUtils.BIG_DECIMAL_SET_SER_DE.serialize(set));
+ }
+
+ @Override
+ public ObjectSet<BigDecimal> deserializeIntermediateResult(CustomObject customObject) {
+ return (ObjectSet<BigDecimal>) ObjectSerDeUtils.BIG_DECIMAL_SET_SER_DE.deserialize(customObject.getBuffer());
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBytesFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBytesFunction.java
new file mode 100644
index 00000000000..4a27c4e6010
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctBytesFunction.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
+import it.unimi.dsi.fastutil.objects.ObjectSet;
+import java.util.Map;
+import org.apache.pinot.common.CustomObject;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.common.ObjectSerDeUtils;
+import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+import org.apache.pinot.spi.utils.ByteArray;
+
+
+public class ArrayAggDistinctBytesFunction extends BaseArrayAggBytesFunction<ObjectSet<ByteArray>> {
+ public ArrayAggDistinctBytesFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, nullHandlingEnabled);
+ }
+
+ @Override
+ public void aggregate(int length, AggregationResultHolder aggregationResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ ObjectOpenHashSet<ByteArray> valueSet =
+ aggregationResultHolder.getResult() != null ? aggregationResultHolder.getResult()
+ : new ObjectOpenHashSet<>(length);
+ if (blockValSet.isSingleValue()) {
+ byte[][] values = blockValSet.getBytesValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ valueSet.add(new ByteArray(values[i]));
+ }
+ });
+ } else {
+ byte[][][] valuesArray = blockValSet.getBytesValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ byte[][] values = valuesArray[i];
+ for (byte[] v : values) {
+ valueSet.add(new ByteArray(v));
+ }
+ }
+ });
+ }
+ aggregationResultHolder.setValue(valueSet);
+ }
+
+ @Override
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, byte[] value) {
+ ObjectOpenHashSet<ByteArray> valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new ObjectOpenHashSet<>();
+ resultHolder.setValueForKey(groupKey, valueSet);
+ }
+ valueSet.add(new ByteArray(value));
+ }
+
+ @Override
+ public ObjectArrayList<ByteArray> extractFinalResult(ObjectSet<ByteArray> intermediateResult) {
+ return intermediateResult == null ? new ObjectArrayList<>() : new ObjectArrayList<>(intermediateResult);
+ }
+
+ @Override
+ public SerializedIntermediateResult serializeIntermediateResult(ObjectSet<ByteArray> set) {
+ return new SerializedIntermediateResult(ObjectSerDeUtils.ObjectType.BytesSet.getValue(),
+ ObjectSerDeUtils.BYTES_SET_SER_DE.serialize(set));
+ }
+
+ @Override
+ public ObjectSet<ByteArray> deserializeIntermediateResult(CustomObject customObject) {
+ return (ObjectSet<ByteArray>) ObjectSerDeUtils.BYTES_SET_SER_DE.deserialize(customObject.getBuffer());
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBigDecimalFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBigDecimalFunction.java
new file mode 100644
index 00000000000..d55afbb7692
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBigDecimalFunction.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectCollection;
+import java.math.BigDecimal;
+import java.util.Map;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public abstract class BaseArrayAggBigDecimalFunction<I extends ObjectCollection<BigDecimal>>
+ extends BaseArrayAggFunction<I, ObjectArrayList<BigDecimal>> {
+ public BaseArrayAggBigDecimalFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, FieldSpec.DataType.BIG_DECIMAL, nullHandlingEnabled);
+ }
+
+ abstract void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, BigDecimal value);
+
+ @Override
+ public void aggregateGroupBySV(int length, int[] groupKeyArray, GroupByResultHolder groupByResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ if (blockValSet.isSingleValue()) {
+ BigDecimal[] values = blockValSet.getBigDecimalValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ setGroupByResult(groupByResultHolder, groupKeyArray[i], values[i]);
+ }
+ });
+ } else {
+ BigDecimal[][] valuesArray = blockValSet.getBigDecimalValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int groupKey = groupKeyArray[i];
+ BigDecimal[] values = valuesArray[i];
+ for (BigDecimal v : values) {
+ setGroupByResult(groupByResultHolder, groupKey, v);
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public void aggregateGroupByMV(int length, int[][] groupKeysArray, GroupByResultHolder groupByResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ if (blockValSet.isSingleValue()) {
+ BigDecimal[] values = blockValSet.getBigDecimalValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int[] groupKeys = groupKeysArray[i];
+ for (int groupKey : groupKeys) {
+ setGroupByResult(groupByResultHolder, groupKey, values[i]);
+ }
+ }
+ });
+ } else {
+ BigDecimal[][] valuesArray = blockValSet.getBigDecimalValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int[] groupKeys = groupKeysArray[i];
+ BigDecimal[] values = valuesArray[i];
+ for (int groupKey : groupKeys) {
+ for (BigDecimal v : values) {
+ setGroupByResult(groupByResultHolder, groupKey, v);
+ }
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public I merge(I intermediateResult1, I intermediateResult2) {
+ if (intermediateResult1 == null) {
+ return intermediateResult2;
+ }
+ if (intermediateResult2 == null) {
+ return intermediateResult1;
+ }
+ intermediateResult1.addAll(intermediateResult2);
+ return intermediateResult1;
+ }
+
+ @Override
+ public ObjectArrayList<BigDecimal> extractFinalResult(I bigDecimals) {
+ if (bigDecimals == null) {
+ return new ObjectArrayList<>();
+ }
+ if (bigDecimals instanceof ObjectArrayList) {
+ return (ObjectArrayList<BigDecimal>) bigDecimals;
+ }
+ return new ObjectArrayList<>(bigDecimals);
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBytesFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBytesFunction.java
new file mode 100644
index 00000000000..8f4660350ca
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggBytesFunction.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.query.aggregation.function.array;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectCollection;
+import java.util.Map;
+import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.utils.ByteArray;
+
+
+/// Shared base for BYTES ARRAY_AGG variants. Both non-distinct (`ObjectArrayList<ByteArray>`) and distinct
+/// (`ObjectOpenHashSet<ByteArray>`) subclasses store `ByteArray` so content-based equality works for the
+/// distinct case and the element type stays consistent with the DataTable/wire layer.
+public abstract class BaseArrayAggBytesFunction<I extends ObjectCollection<ByteArray>>
+ extends BaseArrayAggFunction<I, ObjectArrayList<ByteArray>> {
+ public BaseArrayAggBytesFunction(ExpressionContext expression, boolean nullHandlingEnabled) {
+ super(expression, FieldSpec.DataType.BYTES, nullHandlingEnabled);
+ }
+
+ abstract void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, byte[] value);
+
+ @Override
+ public void aggregateGroupBySV(int length, int[] groupKeyArray, GroupByResultHolder groupByResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ if (blockValSet.isSingleValue()) {
+ byte[][] values = blockValSet.getBytesValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ setGroupByResult(groupByResultHolder, groupKeyArray[i], values[i]);
+ }
+ });
+ } else {
+ byte[][][] valuesArray = blockValSet.getBytesValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int groupKey = groupKeyArray[i];
+ byte[][] values = valuesArray[i];
+ for (byte[] v : values) {
+ setGroupByResult(groupByResultHolder, groupKey, v);
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public void aggregateGroupByMV(int length, int[][] groupKeysArray, GroupByResultHolder groupByResultHolder,
+ Map<ExpressionContext, BlockValSet> blockValSetMap) {
+ BlockValSet blockValSet = blockValSetMap.get(_expression);
+ if (blockValSet.isSingleValue()) {
+ byte[][] values = blockValSet.getBytesValuesSV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int[] groupKeys = groupKeysArray[i];
+ for (int groupKey : groupKeys) {
+ setGroupByResult(groupByResultHolder, groupKey, values[i]);
+ }
+ }
+ });
+ } else {
+ byte[][][] valuesArray = blockValSet.getBytesValuesMV();
+ forEachNotNull(length, blockValSet, (from, to) -> {
+ for (int i = from; i < to; i++) {
+ int[] groupKeys = groupKeysArray[i];
+ byte[][] values = valuesArray[i];
+ for (int groupKey : groupKeys) {
+ for (byte[] v : values) {
+ setGroupByResult(groupByResultHolder, groupKey, v);
+ }
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public I merge(I intermediateResult1, I intermediateResult2) {
+ if (intermediateResult1 == null) {
+ return intermediateResult2;
+ }
+ if (intermediateResult2 == null) {
+ return intermediateResult1;
+ }
+ intermediateResult1.addAll(intermediateResult2);
+ return intermediateResult1;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/ParentAggregationFunctionResultObject.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/ParentAggregationFunctionResultObject.java
index ee441b74e58..987540e9b0b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/ParentAggregationFunctionResultObject.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/ParentAggregationFunctionResultObject.java
@@ -31,7 +31,9 @@ import org.apache.pinot.core.query.utils.rewriter.ParentAggregationResultRewrite
public interface ParentAggregationFunctionResultObject
extends Comparable<ParentAggregationFunctionResultObject>, Serializable {
- // get the nested value of the field at the given row, column
+ /// Get the field from a projection column in the internal form (e.g. `ByteArray` for BYTES, `ByteArray[]` for
+ /// BYTES_ARRAY). Callers that surface the value as a query result must convert to the external form first via
+ /// [DataSchema.ColumnDataType#convert]
Object getField(int rowId, int colId);
// get total number of rows
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxMeasuringValSetWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxMeasuringValSetWrapper.java
index b88e7b2e1ec..1c990253a68 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxMeasuringValSetWrapper.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxMeasuringValSetWrapper.java
@@ -18,7 +18,8 @@
*/
package org.apache.pinot.core.query.aggregation.utils.exprminmax;
-import org.apache.pinot.common.utils.DataSchema;
+import com.google.common.base.Preconditions;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.common.BlockValSet;
@@ -26,52 +27,55 @@ import org.apache.pinot.core.common.BlockValSet;
* Wrapper class for measuring columns in exprmin/max aggregation function.
* Meanly used to do comparison without boxing primitive types.
*/
+@SuppressWarnings({"rawtypes", "unchecked"})
public class ExprMinMaxMeasuringValSetWrapper extends ExprMinMaxWrapperValSet {
- public ExprMinMaxMeasuringValSetWrapper(boolean isSingleValue, DataSchema.ColumnDataType dataType,
- BlockValSet blockValSet) {
- super(dataType, isSingleValue);
+ public ExprMinMaxMeasuringValSetWrapper(BlockValSet blockValSet) {
+ super(resolveStoredType(blockValSet));
setNewBlock(blockValSet);
}
+ private static ColumnDataType resolveStoredType(BlockValSet blockValSet) {
+ Preconditions.checkState(blockValSet.isSingleValue(),
+ "ExprMinMax only supports single-valued measuring columns");
+ return ColumnDataType.fromDataType(blockValSet.getValueType().getStoredType(), true);
+ }
+
public Comparable getComparable(int i) {
- switch (_dataType) {
+ switch (_storedType) {
case INT:
- case BOOLEAN:
return _intValues[i];
case LONG:
- case TIMESTAMP:
return _longValues[i];
case FLOAT:
return _floatValues[i];
case DOUBLE:
return _doublesValues[i];
- case STRING:
case BIG_DECIMAL:
- return (Comparable) _objectsValues[i];
+ case STRING:
+ case BYTES:
+ return _objectsValues[i];
default:
- throw new IllegalStateException("Unsupported data type: " + _dataType);
+ throw new IllegalStateException("Unsupported stored type: " + _storedType);
}
}
- public int compare(int i, Object o) {
- switch (_dataType) {
- case INT:
- case BOOLEAN:
- return Integer.compare((Integer) o, _intValues[i]);
- case LONG:
- case TIMESTAMP:
- return Long.compare((Long) o, _longValues[i]);
- case FLOAT:
- return Float.compare((Float) o, _floatValues[i]);
- case DOUBLE:
- return Double.compare((Double) o, _doublesValues[i]);
- case STRING:
- return ((String) o).compareTo((String) _objectsValues[i]);
- case BIG_DECIMAL:
- return ((java.math.BigDecimal) o).compareTo((java.math.BigDecimal) _objectsValues[i]);
- default:
- throw new IllegalStateException("Unsupported data type in comparison" + _dataType);
- }
+ public int compare(int i, Comparable value) {
+ switch (_storedType) {
+ case INT:
+ return Integer.compare((Integer) value, _intValues[i]);
+ case LONG:
+ return Long.compare((Long) value, _longValues[i]);
+ case FLOAT:
+ return Float.compare((Float) value, _floatValues[i]);
+ case DOUBLE:
+ return Double.compare((Double) value, _doublesValues[i]);
+ case BIG_DECIMAL:
+ case STRING:
+ case BYTES:
+ return value.compareTo(_objectsValues[i]);
+ default:
+ throw new IllegalStateException("Unsupported stored type: " + _storedType);
+ }
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxObject.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxObject.java
index a3220897b81..c34b4228756 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxObject.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxObject.java
@@ -35,6 +35,7 @@ import org.apache.pinot.core.query.aggregation.utils.ParentAggregationFunctionRe
import org.apache.pinot.segment.spi.memory.CompoundDataBuffer;
+@SuppressWarnings("rawtypes")
public class ExprMinMaxObject implements ParentAggregationFunctionResultObject {
// if the object is created but not yet populated, this happens e.g. when a server has no data for
@@ -223,11 +224,9 @@ public class ExprMinMaxObject implements ParentAggregationFunctionResultObject {
for (int i = 0; i < _sizeOfExtremumMeasuringKeys; i++) {
switch (_measuringSchema.getColumnDataType(i)) {
case INT:
- case BOOLEAN:
extremumKeys[i] = _immutableMeasuringKeys.getInt(0, i);
break;
case LONG:
- case TIMESTAMP:
extremumKeys[i] = _immutableMeasuringKeys.getLong(0, i);
break;
case FLOAT:
@@ -236,11 +235,14 @@ public class ExprMinMaxObject implements ParentAggregationFunctionResultObject {
case DOUBLE:
extremumKeys[i] = _immutableMeasuringKeys.getDouble(0, i);
break;
+ case BIG_DECIMAL:
+ extremumKeys[i] = _immutableMeasuringKeys.getBigDecimal(0, i);
+ break;
case STRING:
extremumKeys[i] = _immutableMeasuringKeys.getString(0, i);
break;
- case BIG_DECIMAL:
- extremumKeys[i] = _immutableMeasuringKeys.getBigDecimal(0, i);
+ case BYTES:
+ extremumKeys[i] = _immutableMeasuringKeys.getBytes(0, i);
break;
default:
throw new IllegalStateException("Unsupported data type: " + _measuringSchema.getColumnDataType(i));
@@ -250,45 +252,40 @@ public class ExprMinMaxObject implements ParentAggregationFunctionResultObject {
}
}
- /**
- * Get the field from a projection column
- */
@Override
public Object getField(int rowId, int colId) {
if (_mutable) {
return _extremumProjectionValues.get(rowId)[colId];
} else {
switch (_projectionSchema.getColumnDataType(colId)) {
- case BOOLEAN:
case INT:
return _immutableProjectionVals.getInt(rowId, colId);
- case TIMESTAMP:
case LONG:
return _immutableProjectionVals.getLong(rowId, colId);
case FLOAT:
return _immutableProjectionVals.getFloat(rowId, colId);
case DOUBLE:
return _immutableProjectionVals.getDouble(rowId, colId);
- case JSON:
+ case BIG_DECIMAL:
+ return _immutableProjectionVals.getBigDecimal(rowId, colId);
case STRING:
return _immutableProjectionVals.getString(rowId, colId);
case BYTES:
return _immutableProjectionVals.getBytes(rowId, colId);
- case BIG_DECIMAL:
- return _immutableProjectionVals.getBigDecimal(rowId, colId);
- case BOOLEAN_ARRAY:
case INT_ARRAY:
return _immutableProjectionVals.getIntArray(rowId, colId);
- case TIMESTAMP_ARRAY:
case LONG_ARRAY:
return _immutableProjectionVals.getLongArray(rowId, colId);
case FLOAT_ARRAY:
return _immutableProjectionVals.getFloatArray(rowId, colId);
case DOUBLE_ARRAY:
return _immutableProjectionVals.getDoubleArray(rowId, colId);
+ case BIG_DECIMAL_ARRAY:
+ return _immutableProjectionVals.getBigDecimalArray(rowId, colId);
case STRING_ARRAY:
- case BYTES_ARRAY:
return _immutableProjectionVals.getStringArray(rowId, colId);
+ case BYTES_ARRAY:
+ return _immutableProjectionVals.getBytesArray(rowId, colId);
default:
throw new IllegalStateException("Unsupported data type: " + _projectionSchema.getColumnDataType(colId));
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxProjectionValSetWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxProjectionValSetWrapper.java
index e855c55c9d7..3eb433f231f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxProjectionValSetWrapper.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxProjectionValSetWrapper.java
@@ -18,7 +18,7 @@
*/
package org.apache.pinot.core.query.aggregation.utils.exprminmax;
-import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.common.BlockValSet;
@@ -28,43 +28,39 @@ import org.apache.pinot.core.common.BlockValSet;
*/
public class ExprMinMaxProjectionValSetWrapper extends ExprMinMaxWrapperValSet {
- public ExprMinMaxProjectionValSetWrapper(boolean isSingleValue, DataSchema.ColumnDataType dataType,
- BlockValSet blockValSet) {
- super(dataType, isSingleValue);
+ public ExprMinMaxProjectionValSetWrapper(BlockValSet blockValSet) {
+ super(ColumnDataType.fromDataType(blockValSet.getValueType().getStoredType(), blockValSet.isSingleValue()));
setNewBlock(blockValSet);
}
public Object getValue(int i) {
- switch (_dataType) {
+ switch (_storedType) {
case INT:
- case BOOLEAN:
return _intValues[i];
case LONG:
- case TIMESTAMP:
return _longValues[i];
case FLOAT:
return _floatValues[i];
case DOUBLE:
return _doublesValues[i];
- case STRING:
case BIG_DECIMAL:
+ case STRING:
case BYTES:
- case JSON:
- return _objectsValues[i];
+ return _objectsValues[i];
case INT_ARRAY:
return _intValuesMV[i].length == 0 ? null : _intValuesMV[i];
case LONG_ARRAY:
- case TIMESTAMP_ARRAY:
return _longValuesMV[i].length == 0 ? null : _longValuesMV[i];
case FLOAT_ARRAY:
return _floatValuesMV[i].length == 0 ? null : _floatValuesMV[i];
case DOUBLE_ARRAY:
return _doublesValuesMV[i].length == 0 ? null : _doublesValuesMV[i];
+ case BIG_DECIMAL_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
return _objectsValuesMV[i].length == 0 ? null : _objectsValuesMV[i];
default:
- throw new IllegalStateException("Unsupported data type: " + _dataType);
+ throw new IllegalStateException("Unsupported stored type: " + _storedType);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxWrapperValSet.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxWrapperValSet.java
index 7090e4e9e07..7f1cf0f98f8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxWrapperValSet.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/utils/exprminmax/ExprMinMaxWrapperValSet.java
@@ -18,89 +18,102 @@
*/
package org.apache.pinot.core.query.aggregation.utils.exprminmax;
-import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.common.BlockValSet;
+import org.apache.pinot.spi.utils.ByteArray;
/**
* Wrapper class for the value sets of the column to do exprmin/max on.
* This class is used for type-generic implementation of exprmin/max.
*/
+@SuppressWarnings("rawtypes")
public class ExprMinMaxWrapperValSet {
- protected final DataSchema.ColumnDataType _dataType;
- boolean _isSingleValue;
+ final ColumnDataType _storedType;
int[] _intValues;
long[] _longValues;
float[] _floatValues;
double[] _doublesValues;
- Object[] _objectsValues;
+ Comparable[] _objectsValues;
int[][] _intValuesMV;
long[][] _longValuesMV;
float[][] _floatValuesMV;
double[][] _doublesValuesMV;
- Object[][] _objectsValuesMV;
+ Comparable[][] _objectsValuesMV;
- public ExprMinMaxWrapperValSet(
- DataSchema.ColumnDataType dataType, boolean isSingleValue) {
- _dataType = dataType;
- _isSingleValue = isSingleValue;
+ public ExprMinMaxWrapperValSet(ColumnDataType storedType) {
+ _storedType = storedType;
+ }
+
+ public ColumnDataType getStoredType() {
+ return _storedType;
}
public void setNewBlock(BlockValSet blockValSet) {
- if (_isSingleValue) {
- switch (_dataType) {
- case INT:
- case BOOLEAN:
- _intValues = blockValSet.getIntValuesSV();
- break;
- case LONG:
- case TIMESTAMP:
- _longValues = blockValSet.getLongValuesSV();
- break;
- case FLOAT:
- _floatValues = blockValSet.getFloatValuesSV();
- break;
- case DOUBLE:
- _doublesValues = blockValSet.getDoubleValuesSV();
- break;
- case STRING:
- case JSON:
- _objectsValues = blockValSet.getStringValuesSV();
- break;
- case BIG_DECIMAL:
- _objectsValues = blockValSet.getBigDecimalValuesSV();
- break;
- case BYTES:
- _objectsValues = blockValSet.getBytesValuesSV();
- break;
- default:
- throw new IllegalStateException("Unsupported data type: " + _dataType);
+ switch (_storedType) {
+ case INT:
+ _intValues = blockValSet.getIntValuesSV();
+ break;
+ case LONG:
+ _longValues = blockValSet.getLongValuesSV();
+ break;
+ case FLOAT:
+ _floatValues = blockValSet.getFloatValuesSV();
+ break;
+ case DOUBLE:
+ _doublesValues = blockValSet.getDoubleValuesSV();
+ break;
+ case BIG_DECIMAL:
+ _objectsValues = blockValSet.getBigDecimalValuesSV();
+ break;
+ case STRING:
+ _objectsValues = blockValSet.getStringValuesSV();
+ break;
+ case BYTES: {
+ // Wrap to ByteArray so values stay Comparable and match the DataTable BYTES serialization convention.
+ byte[][] bytesSV = blockValSet.getBytesValuesSV();
+ ByteArray[] byteArrays = new ByteArray[bytesSV.length];
+ for (int i = 0; i < bytesSV.length; i++) {
+ byteArrays[i] = new ByteArray(bytesSV[i]);
+ }
+ _objectsValues = byteArrays;
+ break;
}
- } else {
- switch (_dataType) {
- case INT_ARRAY:
- case BOOLEAN_ARRAY:
- _intValuesMV = blockValSet.getIntValuesMV();
- break;
- case LONG_ARRAY:
- case TIMESTAMP_ARRAY:
- _longValuesMV = blockValSet.getLongValuesMV();
- break;
- case FLOAT_ARRAY:
- _floatValuesMV = blockValSet.getFloatValuesMV();
- break;
- case DOUBLE_ARRAY:
- _doublesValuesMV = blockValSet.getDoubleValuesMV();
- break;
- case STRING_ARRAY:
- _objectsValuesMV = blockValSet.getStringValuesMV();
- break;
- case BYTES_ARRAY:
- _objectsValuesMV = blockValSet.getBytesValuesMV();
- break;
- default:
- throw new IllegalStateException("Unsupported data type: " + _dataType);
+ case INT_ARRAY:
+ _intValuesMV = blockValSet.getIntValuesMV();
+ break;
+ case LONG_ARRAY:
+ _longValuesMV = blockValSet.getLongValuesMV();
+ break;
+ case FLOAT_ARRAY:
+ _floatValuesMV = blockValSet.getFloatValuesMV();
+ break;
+ case DOUBLE_ARRAY:
+ _doublesValuesMV = blockValSet.getDoubleValuesMV();
+ break;
+ case BIG_DECIMAL_ARRAY:
+ _objectsValuesMV = blockValSet.getBigDecimalValuesMV();
+ break;
+ case STRING_ARRAY:
+ _objectsValuesMV = blockValSet.getStringValuesMV();
+ break;
+ case BYTES_ARRAY: {
+ // Wrap to ByteArray[] so values match the DataTable BYTES_ARRAY serialization convention.
+ byte[][][] bytesMV = blockValSet.getBytesValuesMV();
+ ByteArray[][] byteArraysMV = new ByteArray[bytesMV.length][];
+ for (int i = 0; i < bytesMV.length; i++) {
+ byte[][] value = bytesMV[i];
+ ByteArray[] byteArrays = new ByteArray[value.length];
+ for (int j = 0; j < value.length; j++) {
+ byteArrays[j] = new ByteArray(value[j]);
+ }
+ byteArraysMV[i] = byteArrays;
+ }
+ _objectsValuesMV = byteArraysMV;
+ break;
}
+ default:
+ throw new IllegalStateException("Unsupported stored type: " + _storedType);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/BigDecimalDistinctExecutor.java b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/BigDecimalDistinctExecutor.java
index 647e2e7e147..513bb4a031c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/BigDecimalDistinctExecutor.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/BigDecimalDistinctExecutor.java
@@ -50,7 +50,7 @@ public class BigDecimalDistinctExecutor
@Override
protected BigDecimal[][] getValuesMV(BlockValSet blockValSet) {
- throw new UnsupportedOperationException();
+ return blockValSet.getBigDecimalValuesMV();
}
@Override
@@ -77,6 +77,29 @@ public class BigDecimalDistinctExecutor
@Override
protected boolean processMV(BigDecimal[][] values, int from, int to) {
- throw new UnsupportedOperationException();
+ if (_distinctTable.hasLimit()) {
+ if (_distinctTable.hasOrderBy()) {
+ for (int i = from; i < to; i++) {
+ for (BigDecimal value : values[i]) {
+ _distinctTable.addWithOrderBy(value);
+ }
+ }
+ } else {
+ for (int i = from; i < to; i++) {
+ for (BigDecimal value : values[i]) {
+ if (_distinctTable.addWithoutOrderBy(value)) {
+ return true;
+ }
+ }
+ }
+ }
+ } else {
+ for (int i = from; i < to; i++) {
+ for (BigDecimal value : values[i]) {
+ _distinctTable.addUnbounded(value);
+ }
+ }
+ }
+ return false;
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
index b4bf9113206..942544dab50 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
@@ -163,7 +163,7 @@ public class GroupByDataTableReducer implements DataTableReducer {
for (int i = 0; i < numColumns; i++) {
Object value = row[i];
if (value != null) {
- row[i] = columnDataTypes[i].convert(row[i]);
+ row[i] = columnDataTypes[i].convert(value);
}
}
if (havingFilterHandler.isMatch(row)) {
@@ -179,7 +179,7 @@ public class GroupByDataTableReducer implements DataTableReducer {
for (int j = 0; j < numColumns; j++) {
Object value = row[j];
if (value != null) {
- row[j] = columnDataTypes[j].convert(row[j]);
+ row[j] = columnDataTypes[j].convert(value);
}
}
rows.add(row);
@@ -296,9 +296,15 @@ public class GroupByDataTableReducer implements DataTableReducer {
case DOUBLE_ARRAY:
values[colId] = DoubleArrayList.wrap(dataTable.getDoubleArray(rowId, colId));
break;
+ case BIG_DECIMAL_ARRAY:
+ values[colId] = ObjectArrayList.wrap(dataTable.getBigDecimalArray(rowId, colId));
+ break;
case STRING_ARRAY:
values[colId] = ObjectArrayList.wrap(dataTable.getStringArray(rowId, colId));
break;
+ case BYTES_ARRAY:
+ values[colId] = ObjectArrayList.wrap(dataTable.getBytesArray(rowId, colId));
+ break;
case OBJECT:
CustomObject customObject = dataTable.getCustomObject(rowId, colId);
if (customObject != null) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
index 40358ad61d7..d3d1fccea0d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
@@ -437,9 +437,15 @@ public class SelectionOperatorUtils {
case DOUBLE_ARRAY:
dataTableBuilder.setColumn(i, (double[]) columnValue);
break;
+ case BIG_DECIMAL_ARRAY:
+ dataTableBuilder.setColumn(i, (BigDecimal[]) columnValue);
+ break;
case STRING_ARRAY:
dataTableBuilder.setColumn(i, (String[]) columnValue);
break;
+ case BYTES_ARRAY:
+ dataTableBuilder.setColumn(i, (ByteArray[]) columnValue);
+ break;
default:
throw new IllegalStateException("Unsupported data type: " + storedColumnDataTypes + " for column: "
@@ -514,9 +520,15 @@ public class SelectionOperatorUtils {
case DOUBLE_ARRAY:
row[i] = dataTable.getDoubleArray(rowId, i);
break;
+ case BIG_DECIMAL_ARRAY:
+ row[i] = dataTable.getBigDecimalArray(rowId, i);
+ break;
case STRING_ARRAY:
row[i] = dataTable.getStringArray(rowId, i);
break;
+ case BYTES_ARRAY:
+ row[i] = dataTable.getBytesArray(rowId, i);
+ break;
default:
throw new IllegalStateException("Unsupported data type: " + storedColumnDataTypes[i] + " for column: "
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/utils/rewriter/ParentAggregationResultRewriter.java b/pinot-core/src/main/java/org/apache/pinot/core/query/utils/rewriter/ParentAggregationResultRewriter.java
index d27de417527..43099f259ec 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/utils/rewriter/ParentAggregationResultRewriter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/utils/rewriter/ParentAggregationResultRewriter.java
@@ -166,7 +166,10 @@ public class ParentAggregationResultRewriter implements ResultRewriter {
(ParentAggregationFunctionResultObject) row[offset];
// If the parent result has more rows than the current row, extract the value from the row
if (rowIter < parentAggregationFunctionResultObject.getNumberOfRows()) {
- newRow[fieldIter] = parentAggregationFunctionResultObject.getField(rowIter, nestedOffset);
+ // getField returns the value in internal form; convert to external form so downstream format() works
+ // (e.g. ByteArray -> byte[] for BYTES).
+ Object value = parentAggregationFunctionResultObject.getField(rowIter, nestedOffset);
+ newRow[fieldIter] = value != null ? newColumnDataTypes[fieldIter].convert(value) : null;
} else {
newRow[fieldIter] = null;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/DataBlockExtractUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/DataBlockExtractUtils.java
index e8eed72c96b..3e7b3303145 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/DataBlockExtractUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/DataBlockExtractUtils.java
@@ -95,8 +95,12 @@ public final class DataBlockExtractUtils {
return dataBlock.getFloatArray(rowId, colId);
case DOUBLE_ARRAY:
return dataBlock.getDoubleArray(rowId, colId);
+ case BIG_DECIMAL_ARRAY:
+ return dataBlock.getBigDecimalArray(rowId, colId);
case STRING_ARRAY:
return dataBlock.getStringArray(rowId, colId);
+ case BYTES_ARRAY:
+ return dataBlock.getBytesArray(rowId, colId);
// Null
case UNKNOWN:
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/GapfillUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/GapfillUtils.java
index 62dcd180f94..c75bdf15617 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/GapfillUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/GapfillUtils.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.util;
import com.google.common.base.Preconditions;
import java.io.Serializable;
+import java.math.BigDecimal;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -99,6 +100,8 @@ public class GapfillUtils {
return new float[0];
case DOUBLE_ARRAY:
return new double[0];
+ case BIG_DECIMAL_ARRAY:
+ return new BigDecimal[0];
case STRING_ARRAY:
case TIMESTAMP_ARRAY:
return new String[0];
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/SyntheticBlockValSets.java b/pinot-core/src/test/java/org/apache/pinot/core/common/SyntheticBlockValSets.java
index b01e1a1fe72..3b3d38dfc44 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/SyntheticBlockValSets.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/SyntheticBlockValSets.java
@@ -119,6 +119,11 @@ public class SyntheticBlockValSets {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] getStringValuesMV() {
throw new UnsupportedOperationException();
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
index 0a0326bdcc6..44f13638085 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
@@ -23,7 +23,6 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -49,13 +48,25 @@ public class DataBlockBuilderTest {
@DataProvider(name = "columnDataTypes")
ColumnDataType[] columnDataTypes() {
- return EnumSet.complementOf(EnumSet.of(ColumnDataType.BYTES_ARRAY)).toArray(new ColumnDataType[0]);
+ return ColumnDataType.values();
}
@Test(dataProvider = "columnDataTypes")
void testRowBlock(ColumnDataType type)
throws IOException {
- int numRows = 100;
+ runRowBlockTest(type, 100);
+ }
+
+ /// Exercises the multi-batch path of [DataBlockBuilder#buildFromRows] with a row count well past the internal
+ /// `interruptableLoopStep` so that the inner loop runs across multiple `(start, end)` chunks.
+ @Test(dataProvider = "columnDataTypes")
+ void testRowBlockMultiBatch(ColumnDataType type)
+ throws IOException {
+ runRowBlockTest(type, 25_000);
+ }
+
+ private void runRowBlockTest(ColumnDataType type, int numRows)
+ throws IOException {
List<Object[]> rows = generateRows(type, numRows);
DataSchema dataSchema = new DataSchema(new String[]{"column"}, new ColumnDataType[]{type});
AggregationFunction[] aggFunctions = null;
@@ -135,11 +146,26 @@ public class DataBlockBuilderTest {
result.add(new Object[]{new double[]{r.nextDouble(), r.nextDouble()}});
}
break;
+ case BIG_DECIMAL_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ result.add(new Object[]{new BigDecimal[]{BigDecimal.valueOf(r.nextInt()), BigDecimal.valueOf(r.nextInt())}});
+ }
+ break;
case STRING_ARRAY:
for (int i = 0; i < numRows; i++) {
result.add(new Object[]{new String[]{String.valueOf(r.nextInt()), String.valueOf(r.nextInt())}});
}
break;
+ case BYTES_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ result.add(new Object[]{
+ new ByteArray[]{
+ new ByteArray(String.valueOf(r.nextInt()).getBytes()),
+ new ByteArray(String.valueOf(r.nextInt()).getBytes())
+ }
+ });
+ }
+ break;
case OBJECT:
case UNKNOWN:
for (int i = 0; i < numRows; i++) {
@@ -158,7 +184,19 @@ public class DataBlockBuilderTest {
@Test(dataProvider = "columnDataTypes")
void testColumnBlock(ColumnDataType type)
throws IOException {
- int numRows = 100;
+ runColumnBlockTest(type, 100);
+ }
+
+ /// Exercises the multi-batch path of [DataBlockBuilder#buildFromColumns] with a row count past the internal
+ /// `interruptableLoopStep` of `serializeColumnData` so the inner loop runs across multiple `(start, end)` chunks.
+ @Test(dataProvider = "columnDataTypes")
+ void testColumnBlockMultiBatch(ColumnDataType type)
+ throws IOException {
+ runColumnBlockTest(type, 25_000);
+ }
+
+ private void runColumnBlockTest(ColumnDataType type, int numRows)
+ throws IOException {
Object[] column = generateColumns(type, numRows);
DataSchema dataSchema = new DataSchema(new String[]{"column"}, new ColumnDataType[]{type});
AggregationFunction[] aggFunctions = null;
@@ -238,11 +276,24 @@ public class DataBlockBuilderTest {
result[i] = new double[]{r.nextDouble(), r.nextDouble()};
}
break;
+ case BIG_DECIMAL_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ result[i] = new BigDecimal[]{BigDecimal.valueOf(r.nextInt()), BigDecimal.valueOf(r.nextInt())};
+ }
+ break;
case STRING_ARRAY:
for (int i = 0; i < numRows; i++) {
result[i] = new String[]{String.valueOf(r.nextInt()), String.valueOf(r.nextInt())};
}
break;
+ case BYTES_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ result[i] = new ByteArray[]{
+ new ByteArray(String.valueOf(r.nextInt()).getBytes()),
+ new ByteArray(String.valueOf(r.nextInt()).getBytes())
+ };
+ }
+ break;
case OBJECT:
case UNKNOWN:
break;
@@ -354,6 +405,14 @@ public class DataBlockBuilderTest {
}
}
break;
+ case BIG_DECIMAL_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ Object expected = rowToData.apply(i);
+ if (expected != null) {
+ assertEquals(block.getBigDecimalArray(i, 0), expected, "Failure on row " + i);
+ }
+ }
+ break;
case STRING_ARRAY:
for (int i = 0; i < numRows; i++) {
Object expected = rowToData.apply(i);
@@ -362,6 +421,14 @@ public class DataBlockBuilderTest {
}
}
break;
+ case BYTES_ARRAY:
+ for (int i = 0; i < numRows; i++) {
+ Object expected = rowToData.apply(i);
+ if (expected != null) {
+ assertEquals(block.getBytesArray(i, 0), (ByteArray[]) rowToData.apply(i), "Failure on row " + i);
+ }
+ }
+ break;
case OBJECT:
case UNKNOWN:
for (int i = 0; i < numRows; i++) {
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTest.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTest.java
index 24b99912940..696b7d51095 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTest.java
@@ -31,19 +31,18 @@ import org.apache.pinot.common.datablock.RowDataBlock;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.spi.exception.QueryErrorCode;
+import org.apache.pinot.spi.utils.ByteArray;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class DataBlockTest {
- private static final List<ColumnDataType> EXCLUDE_DATA_TYPES =
- List.of(ColumnDataType.OBJECT, ColumnDataType.JSON, ColumnDataType.BYTES, ColumnDataType.BYTES_ARRAY);
+ private static final List<ColumnDataType> EXCLUDE_DATA_TYPES = List.of(ColumnDataType.JSON, ColumnDataType.OBJECT);
private static final int TEST_ROW_COUNT = 5;
@Test
- public void testException()
- throws IOException {
+ public void testException() {
Exception originalException = new UnsupportedOperationException("Expected test exception.");
String expected = "Expected test error";
@@ -61,10 +60,10 @@ public class DataBlockTest {
@Test(dataProvider = "testTypeNullPercentile")
public void testAllDataTypes(int nullPercentile)
- throws Exception {
+ throws IOException {
ColumnDataType[] allDataTypes = ColumnDataType.values();
- List<ColumnDataType> columnDataTypes = new ArrayList<ColumnDataType>();
- List<String> columnNames = new ArrayList<String>();
+ List<ColumnDataType> columnDataTypes = new ArrayList<>();
+ List<String> columnNames = new ArrayList<>();
for (int i = 0; i < allDataTypes.length; i++) {
if (!EXCLUDE_DATA_TYPES.contains(allDataTypes[i])) {
columnNames.add(allDataTypes[i].name());
@@ -115,33 +114,31 @@ public class DataBlockTest {
return new Object[][]{new Object[]{0}, new Object[]{10}, new Object[]{100}};
}
- /**
- * TODO: bytes array serialization probably needs fixing.
- */
- @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = "Unsupported stored type:"
- + " BYTES_ARRAY.*")
- public void bytesArraySerDe()
- throws Exception {
- Object[] row = new Object[1];
- row[0] = new byte[][]{new byte[]{0xD, 0xA}, new byte[]{0xD, 0xA}};
- List<Object[]> rows = new ArrayList<>();
- rows.add(row);
- DataSchema dataSchema = new DataSchema(new String[]{"byteArray"}, new ColumnDataType[]{ColumnDataType.BYTES_ARRAY});
- DataBlockBuilder.buildFromRows(rows, dataSchema);
- }
-
@Test
public void intArraySerDe()
throws IOException {
- Object[] row = new Object[1];
- row[0] = new int[0];
+ Object[] row = new Object[]{new int[0]};
List<Object[]> rows = new ArrayList<>();
rows.add(row);
DataSchema dataSchema = new DataSchema(new String[]{"intArray"}, new ColumnDataType[]{ColumnDataType.INT_ARRAY});
DataBlock dataBlock = DataBlockBuilder.buildFromRows(rows, dataSchema);
- List<ByteBuffer> serialize = dataBlock.serialize();
- DataBlock deserialized = DataBlockUtils.deserialize(serialize);
+ List<ByteBuffer> serialized = dataBlock.serialize();
+ DataBlock deserialized = DataBlockUtils.deserialize(serialized);
int[] intArray = deserialized.getIntArray(0, 0);
Assert.assertEquals(intArray.length, 0);
}
+
+ @Test
+ public void bytesArraySerDe()
+ throws IOException {
+ ByteArray[] expected = new ByteArray[]{new ByteArray(new byte[]{0xD, 0xA}), new ByteArray(new byte[]{0xB, 0xE})};
+ Object[] row = new Object[]{expected};
+ List<Object[]> rows = new ArrayList<>();
+ rows.add(row);
+ DataSchema dataSchema = new DataSchema(new String[]{"byteArray"}, new ColumnDataType[]{ColumnDataType.BYTES_ARRAY});
+ DataBlock dataBlock = DataBlockBuilder.buildFromRows(rows, dataSchema);
+ List<ByteBuffer> serialized = dataBlock.serialize();
+ DataBlock deserialized = DataBlockUtils.deserialize(serialized);
+ Assert.assertEquals(deserialized.getBytesArray(0, 0), expected);
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
index 169b34b2923..37402dc904e 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
@@ -105,13 +105,13 @@ public class DataBlockTestUtils {
}
row[colId] = doubleArray;
break;
- case STRING_ARRAY:
+ case BIG_DECIMAL_ARRAY:
length = RANDOM.nextInt(ARRAY_SIZE);
- String[] stringArray = new String[length];
+ BigDecimal[] bigDecimalArray = new BigDecimal[length];
for (int i = 0; i < length; i++) {
- stringArray[i] = RandomStringUtils.random(RANDOM.nextInt(20));
+ bigDecimalArray[i] = BigDecimal.valueOf(RANDOM.nextDouble());
}
- row[colId] = stringArray;
+ row[colId] = bigDecimalArray;
break;
case BOOLEAN_ARRAY:
length = RANDOM.nextInt(ARRAY_SIZE);
@@ -129,6 +129,24 @@ public class DataBlockTestUtils {
}
row[colId] = timestampArray;
break;
+ case STRING_ARRAY:
+ length = RANDOM.nextInt(ARRAY_SIZE);
+ String[] stringArray = new String[length];
+ for (int i = 0; i < length; i++) {
+ stringArray[i] = RandomStringUtils.random(RANDOM.nextInt(20));
+ }
+ row[colId] = stringArray;
+ break;
+ case BYTES_ARRAY:
+ length = RANDOM.nextInt(ARRAY_SIZE);
+ ByteArray[] bytesArray = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ byte[] bytes = new byte[RANDOM.nextInt(20)];
+ RANDOM.nextBytes(bytes);
+ bytesArray[i] = new ByteArray(bytes);
+ }
+ row[colId] = bytesArray;
+ break;
case MAP:
length = RANDOM.nextInt(ARRAY_SIZE);
Map<String, Object> map = new HashMap<>();
@@ -186,8 +204,12 @@ public class DataBlockTestUtils {
return dataBlock.getFloatArray(rowId, colId);
case DOUBLE_ARRAY:
return dataBlock.getDoubleArray(rowId, colId);
+ case BIG_DECIMAL_ARRAY:
+ return dataBlock.getBigDecimalArray(rowId, colId);
case STRING_ARRAY:
return dataBlock.getStringArray(rowId, colId);
+ case BYTES_ARRAY:
+ return dataBlock.getBytesArray(rowId, colId);
case MAP:
return dataBlock.getMap(rowId, colId);
case UNKNOWN:
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
index a3c8ed94ede..83d5ed7eee4 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.common.datatable;
import java.io.IOException;
import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -67,6 +68,8 @@ public class DataTableSerDeTest {
private static final int[][] BOOLEAN_ARRAYS = new int[NUM_ROWS][];
private static final long[][] TIMESTAMP_ARRAYS = new long[NUM_ROWS][];
private static final String[][] STRING_ARRAYS = new String[NUM_ROWS][];
+ private static final ByteArray[][] BYTES_ARRAYS = new ByteArray[NUM_ROWS][];
+ private static final BigDecimal[][] BIG_DECIMAL_ARRAYS = new BigDecimal[NUM_ROWS][];
private static final Map<String, Object>[] MAPS = new Map[NUM_ROWS];
@Test(dataProvider = "versionProvider")
@@ -400,6 +403,15 @@ public class DataTableSerDeTest {
DOUBLE_ARRAYS[rowId] = doubleArray;
dataTableBuilder.setColumn(colId, doubleArray);
break;
+ case BIG_DECIMAL_ARRAY:
+ length = RANDOM.nextInt(20);
+ BigDecimal[] bigDecimalArray = new BigDecimal[length];
+ for (int i = 0; i < length; i++) {
+ bigDecimalArray[i] = BigDecimal.valueOf(RANDOM.nextDouble());
+ }
+ BIG_DECIMAL_ARRAYS[rowId] = bigDecimalArray;
+ dataTableBuilder.setColumn(colId, bigDecimalArray);
+ break;
case BOOLEAN_ARRAY:
length = RANDOM.nextInt(2);
int[] booleanArray = new int[length];
@@ -419,7 +431,14 @@ public class DataTableSerDeTest {
dataTableBuilder.setColumn(colId, timestampArray);
break;
case BYTES_ARRAY:
- // TODO: add once implementation of datatable bytes array support is added
+ length = RANDOM.nextInt(20);
+ ByteArray[] bytesArray = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ bytesArray[i] =
+ new ByteArray(RandomStringUtils.random(RANDOM.nextInt(20)).getBytes(StandardCharsets.UTF_8));
+ }
+ BYTES_ARRAYS[rowId] = bytesArray;
+ dataTableBuilder.setColumn(colId, bytesArray);
break;
case STRING_ARRAY:
length = RANDOM.nextInt(20);
@@ -516,6 +535,10 @@ public class DataTableSerDeTest {
Assert.assertTrue(Arrays.equals(newDataTable.getDoubleArray(rowId, colId), DOUBLE_ARRAYS[rowId]),
ERROR_MESSAGE);
break;
+ case BIG_DECIMAL_ARRAY:
+ Assert.assertTrue(Arrays.equals(newDataTable.getBigDecimalArray(rowId, colId), BIG_DECIMAL_ARRAYS[rowId]),
+ ERROR_MESSAGE);
+ break;
case BOOLEAN_ARRAY:
Assert.assertTrue(Arrays.equals(newDataTable.getIntArray(rowId, colId), BOOLEAN_ARRAYS[rowId]),
ERROR_MESSAGE);
@@ -525,7 +548,8 @@ public class DataTableSerDeTest {
ERROR_MESSAGE);
break;
case BYTES_ARRAY:
- // TODO: add once implementation of datatable bytes array support is added
+ Assert.assertTrue(Arrays.equals(newDataTable.getBytesArray(rowId, colId), BYTES_ARRAYS[rowId]),
+ ERROR_MESSAGE);
break;
case STRING_ARRAY:
Assert.assertTrue(Arrays.equals(newDataTable.getStringArray(rowId, colId), STRING_ARRAYS[rowId]),
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
index 56d1bf7595d..86855f564db 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
@@ -992,6 +992,11 @@ public class DateTimeConversionTransformFunctionTest extends BaseTransformFuncti
return new double[0][];
}
+ @Override
+ public BigDecimal[][] transformToBigDecimalValuesMV(ValueBlock valueBlock) {
+ return new BigDecimal[0][];
+ }
+
@Override
public String[][] transformToStringValuesMV(ValueBlock valueBlock) {
return new String[0][];
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/ArrayAggFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/ArrayAggFunctionTest.java
index 6a815aaeba9..7cd704a3100 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/ArrayAggFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/ArrayAggFunctionTest.java
@@ -514,6 +514,32 @@ public class ArrayAggFunctionTest extends AbstractAggregationFunctionTest {
);
}
+ @Test
+ void aggregationBigDecimalWithNullHandlingEnabled() {
+ new DataTypeScenario(FieldSpec.DataType.BIG_DECIMAL).getDeclaringTable(true)
+ .onFirstInstance("myField",
+ "1.5",
+ "null",
+ "2.5"
+ ).andOnSecondInstance("myField",
+ "1.5",
+ "null",
+ "2.5"
+ ).whenQuery("select arrayagg(myField, 'BIG_DECIMAL') from testTable")
+ .thenResultIs(new Object[]{new String[]{"1.5", "2.5", "1.5", "2.5"}});
+ }
+
+ @Test
+ void aggregationDistinctBigDecimalWithNullHandlingEnabled() {
+ new DataTypeScenario(FieldSpec.DataType.BIG_DECIMAL).getDeclaringTable(true)
+ .onFirstInstance("myField",
+ "1.5",
+ "null",
+ "1.5"
+ ).whenQuery("select arrayagg(myField, 'BIG_DECIMAL', true) from testTable")
+ .thenResultIs(new Object[]{new String[]{"1.5"}});
+ }
+
@Test
void aggregationStringWithNullHandlingDisabled() {
new DataTypeScenario(FieldSpec.DataType.STRING).getDeclaringTable(false)
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunctionTest.java
index 56464fb30e4..ee67a9f872c 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunctionTest.java
@@ -184,6 +184,115 @@ public class DistinctCountAggregationFunctionTest extends AbstractAggregationFun
.thenResultIs("INTEGER", "2");
}
+ @Test
+ public void distinctCountBigDecimal() {
+ FluentQueryTest.withBaseDir(_baseDir)
+ .givenTable(SINGLE_FIELD_NULLABLE_DIMENSION_SCHEMAS.get(FieldSpec.DataType.BIG_DECIMAL),
+ SINGLE_FIELD_TABLE_CONFIG)
+ .onFirstInstance(
+ "myField",
+ "1.5",
+ "2.5"
+ )
+ .andOnSecondInstance(
+ "myField",
+ "2.5",
+ "3.5"
+ )
+ .whenQuery("select DISTINCT_COUNT(myField) from testTable")
+ .thenResultIs("INTEGER", "3");
+ }
+
+ @Test
+ public void distinctCountBigDecimalGroupBy() {
+ // Schema column order is alphabetical via TreeSet, so row values must be in {grp, value} order.
+ FluentQueryTest.withBaseDir(_baseDir)
+ .givenTable(
+ new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addSingleValueDimension("value", FieldSpec.DataType.BIG_DECIMAL)
+ .addSingleValueDimension("grp", FieldSpec.DataType.STRING)
+ .build(), SINGLE_FIELD_TABLE_CONFIG)
+ .onFirstInstance(
+ new Object[]{"a", "1.5"},
+ new Object[]{"a", "2.5"}
+ )
+ .andOnSecondInstance(
+ new Object[]{"a", "2.5"},
+ new Object[]{"b", "3.5"}
+ )
+ .whenQuery("select grp, DISTINCT_COUNT(value) from testTable group by grp order by grp")
+ .thenResultIs("STRING | INTEGER",
+ "a | 2",
+ "b | 1"
+ );
+ }
+
+ @Test
+ public void distinctCountBigDecimalMV() {
+ FluentQueryTest.withBaseDir(_baseDir)
+ .givenTable(
+ new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addMultiValueDimension("mv", FieldSpec.DataType.BIG_DECIMAL)
+ .build(), SINGLE_FIELD_TABLE_CONFIG)
+ .onFirstInstance(
+ new Object[]{"1.5;2.5"},
+ new Object[]{"2.5;3.5"}
+ )
+ .andOnSecondInstance(
+ new Object[]{"3.5;4.5"}
+ )
+ .whenQuery("select DISTINCT_COUNT(mv) from testTable")
+ .thenResultIs("INTEGER", "4");
+ }
+
+ @Test
+ public void distinctCountBigDecimalMVGroupBySV() {
+ // Schema column order is alphabetical via TreeSet, so row values must be in {grp, mv} order.
+ FluentQueryTest.withBaseDir(_baseDir)
+ .givenTable(
+ new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addMultiValueDimension("mv", FieldSpec.DataType.BIG_DECIMAL)
+ .addSingleValueDimension("grp", FieldSpec.DataType.STRING)
+ .build(), SINGLE_FIELD_TABLE_CONFIG)
+ .onFirstInstance(
+ new Object[]{"a", "1.5;2.5"},
+ new Object[]{"a", "2.5;3.5"}
+ )
+ .andOnSecondInstance(
+ new Object[]{"b", "3.5;4.5"}
+ )
+ .whenQuery("select grp, DISTINCT_COUNT(mv) from testTable group by grp order by grp")
+ .thenResultIs("STRING | INTEGER",
+ "a | 3",
+ "b | 2"
+ );
+ }
+
+ @Test
+ public void distinctCountBigDecimalMVGroupByMV() {
+ // Schema column order is alphabetical via TreeSet, so row values must be in {grp, mv} order.
+ FluentQueryTest.withBaseDir(_baseDir)
+ .givenTable(
+ new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addMultiValueDimension("mv", FieldSpec.DataType.BIG_DECIMAL)
+ .addMultiValueDimension("grp", FieldSpec.DataType.STRING)
+ .build(), SINGLE_FIELD_TABLE_CONFIG)
+ .onFirstInstance(
+ new Object[]{"a;b", "1.5;2.5"},
+ new Object[]{"b;c", "2.5;3.5"}
+ )
+ .whenQuery("select grp, DISTINCT_COUNT(mv) from testTable group by grp order by grp")
+ .thenResultIs("STRING | INTEGER",
+ "a | 2",
+ "b | 3",
+ "c | 2"
+ );
+ }
+
@Test
public void distinctCountMVGroupByMVWithNulls() {
FluentQueryTest.withBaseDir(_baseDir)
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
index a37f9fe6351..5788d20eda9 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/util/SchemaUtilsTest.java
@@ -292,31 +292,12 @@ public class SchemaUtilsTest {
@Test
public void testValidateMultiValueFieldSpec() {
- Schema pinotSchema;
-
- pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .addMultiValueDimension("myJsonCol", FieldSpec.DataType.JSON).build();
-
- checkValidationFails(pinotSchema);
-
- pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .addMultiValueDimension("myBigDecimalCol", FieldSpec.DataType.BIG_DECIMAL).build();
-
+ Schema pinotSchema = new Schema.SchemaBuilder()
+ .setSchemaName(TABLE_NAME)
+ .addSingleValueDimension("myCol", DataType.STRING)
+ .addMultiValueDimension("myJsonCol", DataType.JSON)
+ .build();
checkValidationFails(pinotSchema);
-
- pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .addSingleValueDimension("myJsonCol", FieldSpec.DataType.JSON).build();
-
- SchemaUtils.validate(pinotSchema);
-
- pinotSchema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
- .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
- .addSingleValueDimension("myBigDecimalCol", FieldSpec.DataType.BIG_DECIMAL).build();
-
- SchemaUtils.validate(pinotSchema);
}
@Test
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
index dc5e7101a35..bd724a2141e 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java
@@ -90,12 +90,16 @@ public class DistinctQueriesTest extends BaseQueriesTest {
private static final String LONG_MV_COLUMN = "longMVColumn";
private static final String FLOAT_MV_COLUMN = "floatMVColumn";
private static final String DOUBLE_MV_COLUMN = "doubleMVColumn";
+ private static final String BIG_DECIMAL_MV_COLUMN = "bigDecimalMVColumn";
private static final String STRING_MV_COLUMN = "stringMVColumn";
+ private static final String BYTES_MV_COLUMN = "bytesMVColumn";
private static final String RAW_INT_MV_COLUMN = "rawIntMVColumn";
private static final String RAW_LONG_MV_COLUMN = "rawLongMVColumn";
private static final String RAW_FLOAT_MV_COLUMN = "rawFloatMVColumn";
private static final String RAW_DOUBLE_MV_COLUMN = "rawDoubleMVColumn";
+ private static final String RAW_BIG_DECIMAL_MV_COLUMN = "rawBigDecimalMVColumn";
private static final String RAW_STRING_MV_COLUMN = "rawStringMVColumn";
+ private static final String RAW_BYTES_MV_COLUMN = "rawBytesMVColumn";
//@formatter:off
private static final Schema SCHEMA = new Schema.SchemaBuilder()
@@ -117,12 +121,16 @@ public class DistinctQueriesTest extends BaseQueriesTest {
.addMultiValueDimension(LONG_MV_COLUMN, DataType.LONG)
.addMultiValueDimension(FLOAT_MV_COLUMN, DataType.FLOAT)
.addMultiValueDimension(DOUBLE_MV_COLUMN, DataType.DOUBLE)
+ .addMultiValueDimension(BIG_DECIMAL_MV_COLUMN, DataType.BIG_DECIMAL)
.addMultiValueDimension(STRING_MV_COLUMN, DataType.STRING)
+ .addMultiValueDimension(BYTES_MV_COLUMN, DataType.BYTES)
.addMultiValueDimension(RAW_INT_MV_COLUMN, DataType.INT)
.addMultiValueDimension(RAW_LONG_MV_COLUMN, DataType.LONG)
.addMultiValueDimension(RAW_FLOAT_MV_COLUMN, DataType.FLOAT)
.addMultiValueDimension(RAW_DOUBLE_MV_COLUMN, DataType.DOUBLE)
+ .addMultiValueDimension(RAW_BIG_DECIMAL_MV_COLUMN, DataType.BIG_DECIMAL)
.addMultiValueDimension(RAW_STRING_MV_COLUMN, DataType.STRING)
+ .addMultiValueDimension(RAW_BYTES_MV_COLUMN, DataType.BYTES)
.build();
//@formatter:on
@@ -130,7 +138,7 @@ public class DistinctQueriesTest extends BaseQueriesTest {
.setNoDictionaryColumns(
Arrays.asList(RAW_INT_COLUMN, RAW_LONG_COLUMN, RAW_FLOAT_COLUMN, RAW_DOUBLE_COLUMN, RAW_BIG_DECIMAL_COLUMN,
RAW_STRING_COLUMN, RAW_BYTES_COLUMN, RAW_INT_MV_COLUMN, RAW_LONG_MV_COLUMN, RAW_FLOAT_MV_COLUMN,
- RAW_DOUBLE_MV_COLUMN, RAW_STRING_MV_COLUMN))
+ RAW_DOUBLE_MV_COLUMN, RAW_BIG_DECIMAL_MV_COLUMN, RAW_STRING_MV_COLUMN, RAW_BYTES_MV_COLUMN))
.build();
private IndexSegment _indexSegment;
@@ -197,16 +205,26 @@ public class DistinctQueriesTest extends BaseQueriesTest {
record.putValue(RAW_STRING_COLUMN, value);
record.putValue(RAW_BYTES_COLUMN, Integer.toString(value).getBytes(UTF_8));
Integer[] mvValue = new Integer[]{value, value + NUM_UNIQUE_RECORDS_PER_SEGMENT};
+ BigDecimal[] bigDecimalMVValue =
+ new BigDecimal[]{BigDecimal.valueOf(value), BigDecimal.valueOf(value + NUM_UNIQUE_RECORDS_PER_SEGMENT)};
+ byte[][] bytesMVValue = new byte[][]{
+ StringUtils.leftPad(Integer.toString(value), 4).getBytes(UTF_8),
+ StringUtils.leftPad(Integer.toString(value + NUM_UNIQUE_RECORDS_PER_SEGMENT), 4).getBytes(UTF_8)
+ };
record.putValue(INT_MV_COLUMN, mvValue);
record.putValue(LONG_MV_COLUMN, mvValue);
record.putValue(FLOAT_MV_COLUMN, mvValue);
record.putValue(DOUBLE_MV_COLUMN, mvValue);
+ record.putValue(BIG_DECIMAL_MV_COLUMN, bigDecimalMVValue);
record.putValue(STRING_MV_COLUMN, mvValue);
+ record.putValue(BYTES_MV_COLUMN, bytesMVValue);
record.putValue(RAW_INT_MV_COLUMN, mvValue);
record.putValue(RAW_LONG_MV_COLUMN, mvValue);
record.putValue(RAW_FLOAT_MV_COLUMN, mvValue);
record.putValue(RAW_DOUBLE_MV_COLUMN, mvValue);
+ record.putValue(RAW_BIG_DECIMAL_MV_COLUMN, bigDecimalMVValue);
record.putValue(RAW_STRING_MV_COLUMN, mvValue);
+ record.putValue(RAW_BYTES_MV_COLUMN, bytesMVValue);
uniqueRecords.add(record);
}
@@ -253,7 +271,8 @@ public class DistinctQueriesTest extends BaseQueriesTest {
"SELECT DISTINCT(intMVColumn) FROM testTable",
"SELECT DISTINCT(longMVColumn) FROM testTable",
"SELECT DISTINCT(floatMVColumn) FROM testTable",
- "SELECT DISTINCT(doubleMVColumn) FROM testTable"
+ "SELECT DISTINCT(doubleMVColumn) FROM testTable",
+ "SELECT DISTINCT(bigDecimalMVColumn) FROM testTable"
);
//@formatter:on
// Query should be solved with dictionary, so it should return the 10 smallest values
@@ -325,7 +344,8 @@ public class DistinctQueriesTest extends BaseQueriesTest {
//@formatter:off
List<String> queries = Arrays.asList(
"SELECT DISTINCT(bytesColumn) FROM testTable",
- "SELECT DISTINCT(rawBytesColumn) FROM testTable"
+ "SELECT DISTINCT(rawBytesColumn) FROM testTable",
+ "SELECT DISTINCT(bytesMVColumn) FROM testTable"
);
//@formatter:on
Set<Integer> expectedValues = new HashSet<>();
@@ -351,7 +371,8 @@ public class DistinctQueriesTest extends BaseQueriesTest {
"SELECT DISTINCT(rawIntMVColumn) FROM testTable",
"SELECT DISTINCT(rawLongMVColumn) FROM testTable",
"SELECT DISTINCT(rawFloatMVColumn) FROM testTable",
- "SELECT DISTINCT(rawDoubleMVColumn) FROM testTable"
+ "SELECT DISTINCT(rawDoubleMVColumn) FROM testTable",
+ "SELECT DISTINCT(rawBigDecimalMVColumn) FROM testTable"
);
//@formatter:on
// We define a specific result set here since the data read from raw is in the order added
@@ -385,6 +406,23 @@ public class DistinctQueriesTest extends BaseQueriesTest {
}
assertEquals(actualValues, expectedValues);
}
+ {
+ // Raw MV bytes column
+ //@formatter:off
+ String query = "SELECT DISTINCT(rawBytesMVColumn) FROM testTable";
+ //@formatter:on
+ // We define a specific result set here since the data read from raw is in the order added
+ Set<Integer> expectedValues = new HashSet<>(Arrays.asList(0, 1, 2, 3, 4, 100, 101, 102, 103, 104));
+ DistinctTable distinctTable = getDistinctTableInnerSegment(query);
+ assertEquals(distinctTable.size(), 10);
+ Set<Integer> actualValues = new HashSet<>();
+ for (Object[] values : distinctTable.getRows()) {
+ assertEquals(values.length, 1);
+ assertTrue(values[0] instanceof ByteArray);
+ actualValues.add(Integer.parseInt(new String(((ByteArray) values[0]).getBytes(), UTF_8).trim()));
+ }
+ assertEquals(actualValues, expectedValues);
+ }
}
@Test
@@ -431,7 +469,8 @@ public class DistinctQueriesTest extends BaseQueriesTest {
"SELECT DISTINCT(intMVColumn) FROM testTable ORDER BY intMVColumn",
"SELECT DISTINCT(longMVColumn) FROM testTable ORDER BY longMVColumn",
"SELECT DISTINCT(floatMVColumn) FROM testTable ORDER BY floatMVColumn",
- "SELECT DISTINCT(doubleMVColumn) FROM testTable ORDER BY doubleMVColumn"
+ "SELECT DISTINCT(doubleMVColumn) FROM testTable ORDER BY doubleMVColumn",
+ "SELECT DISTINCT(bigDecimalMVColumn) FROM testTable ORDER BY bigDecimalMVColumn"
);
//@formatter:on
Set<Integer> expectedValues = new HashSet<>();
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/ExprMinMaxTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/ExprMinMaxTest.java
index f78cecf303c..e0155e6e0d4 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/ExprMinMaxTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/ExprMinMaxTest.java
@@ -72,33 +72,38 @@ public class ExprMinMaxTest extends BaseQueriesTest {
private static final String LONG_COLUMN = "longColumn";
private static final String FLOAT_COLUMN = "floatColumn";
private static final String DOUBLE_COLUMN = "doubleColumn";
- private static final String MV_DOUBLE_COLUMN = "mvDoubleColumn";
+ private static final String BIG_DECIMAL_COLUMN = "bigDecimalColumn";
+ private static final String BOOLEAN_COLUMN = "booleanColumn";
+ private static final String TIMESTAMP_COLUMN = "timestampColumn";
+ private static final String STRING_COLUMN = "stringColumn";
+ private static final String JSON_COLUMN = "jsonColumn";
+ private static final String BYTES_COLUMN = "bytesColumn";
private static final String MV_INT_COLUMN = "mvIntColumn";
- private static final String MV_BYTES_COLUMN = "mvBytesColumn";
+ private static final String MV_DOUBLE_COLUMN = "mvDoubleColumn";
private static final String MV_STRING_COLUMN = "mvStringColumn";
- private static final String STRING_COLUMN = "stringColumn";
+ private static final String MV_BYTES_COLUMN = "mvBytesColumn";
private static final String GROUP_BY_INT_COLUMN = "groupByIntColumn";
private static final String GROUP_BY_MV_INT_COLUMN = "groupByMVIntColumn";
private static final String GROUP_BY_INT_COLUMN2 = "groupByIntColumn2";
- private static final String BIG_DECIMAL_COLUMN = "bigDecimalColumn";
- private static final String TIMESTAMP_COLUMN = "timestampColumn";
- private static final String BOOLEAN_COLUMN = "booleanColumn";
- private static final String JSON_COLUMN = "jsonColumn";
- private static final Schema SCHEMA = new Schema.SchemaBuilder().addSingleValueDimension(INT_COLUMN, DataType.INT)
- .addSingleValueDimension(LONG_COLUMN, DataType.LONG).addSingleValueDimension(FLOAT_COLUMN, DataType.FLOAT)
- .addSingleValueDimension(DOUBLE_COLUMN, DataType.DOUBLE).addMultiValueDimension(MV_INT_COLUMN, DataType.INT)
- .addMultiValueDimension(MV_BYTES_COLUMN, DataType.BYTES)
- .addMultiValueDimension(MV_STRING_COLUMN, DataType.STRING)
+ private static final Schema SCHEMA = new Schema.SchemaBuilder()
+ .addSingleValueDimension(INT_COLUMN, DataType.INT)
+ .addSingleValueDimension(LONG_COLUMN, DataType.LONG)
+ .addSingleValueDimension(FLOAT_COLUMN, DataType.FLOAT)
+ .addSingleValueDimension(DOUBLE_COLUMN, DataType.DOUBLE)
+ .addSingleValueDimension(BIG_DECIMAL_COLUMN, DataType.BIG_DECIMAL)
+ .addSingleValueDimension(BOOLEAN_COLUMN, DataType.BOOLEAN)
+ .addSingleValueDimension(TIMESTAMP_COLUMN, DataType.TIMESTAMP)
.addSingleValueDimension(STRING_COLUMN, DataType.STRING)
+ .addSingleValueDimension(JSON_COLUMN, DataType.JSON)
+ .addSingleValueDimension(BYTES_COLUMN, DataType.BYTES)
+ .addMultiValueDimension(MV_INT_COLUMN, DataType.INT)
+ .addMultiValueDimension(MV_DOUBLE_COLUMN, DataType.DOUBLE)
+ .addMultiValueDimension(MV_STRING_COLUMN, DataType.STRING)
+ .addMultiValueDimension(MV_BYTES_COLUMN, DataType.BYTES)
.addSingleValueDimension(GROUP_BY_INT_COLUMN, DataType.INT)
.addMultiValueDimension(GROUP_BY_MV_INT_COLUMN, DataType.INT)
.addSingleValueDimension(GROUP_BY_INT_COLUMN2, DataType.INT)
- .addSingleValueDimension(BIG_DECIMAL_COLUMN, DataType.BIG_DECIMAL)
- .addSingleValueDimension(TIMESTAMP_COLUMN, DataType.TIMESTAMP)
- .addSingleValueDimension(BOOLEAN_COLUMN, DataType.BOOLEAN)
- .addMultiValueDimension(MV_DOUBLE_COLUMN, DataType.DOUBLE)
- .addSingleValueDimension(JSON_COLUMN, DataType.JSON)
.build();
private static final TableConfig TABLE_CONFIG =
new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
@@ -135,26 +140,27 @@ public class ExprMinMaxTest extends BaseQueriesTest {
record.putValue(LONG_COLUMN, (long) i - NUM_RECORDS / 2);
record.putValue(FLOAT_COLUMN, (float) i * 0.5);
record.putValue(DOUBLE_COLUMN, (double) i);
- record.putValue(MV_INT_COLUMN, Arrays.asList(i, i + 1, i + 2));
- record.putValue(MV_BYTES_COLUMN, Arrays.asList(String.valueOf(i).getBytes(), String.valueOf(i + 1).getBytes(),
- String.valueOf(i + 2).getBytes()));
- record.putValue(MV_STRING_COLUMN, Arrays.asList("a" + i, "a" + i + 1, "a" + i + 2));
+ record.putValue(BIG_DECIMAL_COLUMN, new BigDecimal(-i * i + 1200 * i));
+ record.putValue(BOOLEAN_COLUMN, i % 2);
+ record.putValue(TIMESTAMP_COLUMN, 1683138373879L - i);
if (i < 20) {
record.putValue(STRING_COLUMN, stringSVVals[i % stringSVVals.length]);
} else {
record.putValue(STRING_COLUMN, "a33");
}
+ record.putValue(JSON_COLUMN, "{\"name\":\"John\", \"age\":" + i + ", \"car\":null}");
+ record.putValue(BYTES_COLUMN, String.valueOf(i).getBytes());
+ record.putValue(MV_INT_COLUMN, Arrays.asList(i, i + 1, i + 2));
+ record.putValue(MV_DOUBLE_COLUMN, Arrays.asList((double) i, (double) i * i, (double) i * i * i));
+ record.putValue(MV_STRING_COLUMN, Arrays.asList("a" + i, "a" + i + 1, "a" + i + 2));
+ record.putValue(MV_BYTES_COLUMN, Arrays.asList(String.valueOf(i).getBytes(), String.valueOf(i + 1).getBytes(),
+ String.valueOf(i + 2).getBytes()));
record.putValue(GROUP_BY_INT_COLUMN, i % 5);
record.putValue(GROUP_BY_MV_INT_COLUMN, Arrays.asList(i % 10, (i + 1) % 10));
if (i == j) {
j *= 2;
}
record.putValue(GROUP_BY_INT_COLUMN2, j);
- record.putValue(BIG_DECIMAL_COLUMN, new BigDecimal(-i * i + 1200 * i));
- record.putValue(TIMESTAMP_COLUMN, 1683138373879L - i);
- record.putValue(BOOLEAN_COLUMN, i % 2);
- record.putValue(MV_DOUBLE_COLUMN, Arrays.asList((double) i, (double) i * i, (double) i * i * i));
- record.putValue(JSON_COLUMN, "{\"name\":\"John\", \"age\":" + i + ", \"car\":null}");
records.add(record);
}
@@ -200,12 +206,6 @@ public class ExprMinMaxTest extends BaseQueriesTest {
Assert.assertTrue(brokerResponse.getExceptions().get(0).getMessage().contains(
"ExprMinMax only supports single-valued measuring columns"
));
-
- query = "SELECT expr_max(mvDoubleColumn, jsonColumn) FROM testTable";
- brokerResponse = getBrokerResponse(query);
- Assert.assertTrue(brokerResponse.getExceptions().get(0).getMessage().contains(
- "Cannot compute exprminMax measuring on non-comparable type: JSON"
- ));
}
@Test
@@ -359,16 +359,26 @@ public class ExprMinMaxTest extends BaseQueriesTest {
assertEquals(rows.get(i)[1], "a");
}
- // TODO: The following query throws an exception,
- // requires fix for multi-value bytes column serialization in DataBlock
- query = "SELECT expr_min(mvBytesColumn, intColumn) FROM testTable";
+ // SV BYTES projection: expr_min picks the row with min(intColumn) = 0 (i.e. i=0), returning
+ // bytesColumn at that row = bytes("0"). Formatted as hex by DataSchema.
+ query = "SELECT expr_min(bytesColumn, intColumn) FROM testTable";
+ brokerResponse = getBrokerResponse(query);
+ resultTable = brokerResponse.getResultTable();
+ rows = resultTable.getRows();
+ assertEquals(rows.size(), 2);
+ assertEquals(rows.get(0)[0], "30");
+ assertEquals(rows.get(1)[0], "30");
- try {
- getBrokerResponse(query);
- fail("remove this test case, now mvBytesColumn works correctly in serialization");
- } catch (Exception e) {
- assertTrue(e.getMessage().contains("Unsupported stored type: BYTES_ARRAY"));
- }
+ // MV BYTES projection: expr_min picks the row with min(intColumn) = 0 (i.e. i=0), returning
+ // mvBytesColumn at that row = [bytes("0"), bytes("1"), bytes("2")]. Formatted as hex by DataSchema.
+ query = "SELECT expr_min(mvBytesColumn, intColumn) FROM testTable";
+ brokerResponse = getBrokerResponse(query);
+ resultTable = brokerResponse.getResultTable();
+ rows = resultTable.getRows();
+ assertEquals(rows.size(), 2);
+ String[] expectedMvBytes = {"30", "31", "32"};
+ assertEquals(rows.get(0)[0], expectedMvBytes);
+ assertEquals(rows.get(1)[0], expectedMvBytes);
}
@Test
diff --git a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
index f9d95795cdb..01b98fdd3da 100644
--- a/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
+++ b/pinot-integration-test-base/src/test/java/org/apache/pinot/integration/tests/ClusterTest.java
@@ -758,8 +758,9 @@ public abstract class ClusterTest extends ControllerTest {
case DOUBLE_ARRAY:
array[k] = jsonValue.get(k).asDouble();
break;
- case STRING_ARRAY:
+ case BIG_DECIMAL_ARRAY:
case TIMESTAMP_ARRAY:
+ case STRING_ARRAY:
case BYTES_ARRAY:
array[k] = jsonValue.get(k).textValue();
break;
@@ -791,11 +792,11 @@ public abstract class ClusterTest extends ControllerTest {
case DOUBLE:
object = jsonValue.asDouble();
break;
+ case BIG_DECIMAL:
+ case TIMESTAMP:
case STRING:
case BYTES:
- case TIMESTAMP:
case JSON:
- case BIG_DECIMAL:
object = jsonValue.textValue();
break;
case UNKNOWN:
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
index dad2bfbff29..739058d5945 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
@@ -19,15 +19,15 @@
package org.apache.pinot.integration.tests.custom;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.avro.Schema;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.Schema;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -37,7 +37,6 @@ import static org.testng.Assert.assertTrue;
@Test(suiteName = "CustomClusterIntegrationTest")
public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
-
private static final String DEFAULT_TABLE_NAME = "ArrayTest";
private static final String BOOLEAN_COLUMN = "boolCol";
private static final String BOOLEAN_FROM_INT_COLUMN = "boolColFromInt";
@@ -54,7 +53,9 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
private static final String BOOLEAN_FROM_STRING_ARRAY_COLUMN = "booleanArrayColFromStringArray";
private static final String LONG_ARRAY_COLUMN = "longArrayCol";
private static final String DOUBLE_ARRAY_COLUMN = "doubleArrayCol";
+ private static final String BIG_DECIMAL_ARRAY_COLUMN = "bigDecimalArrayCol";
private static final String STRING_ARRAY_COLUMN = "stringArrayCol";
+ private static final String BYTES_ARRAY_COLUMN = "bytesArrayCol";
@Override
protected long getCountStarResult() {
@@ -728,6 +729,38 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).asLong(), getCountStarResult());
}
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArrayLengthBigDecimalAndBytes(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // Each row's bigDecimalArrayCol and bytesArrayCol contains 4 elements.
+ String queryBigDecimal = String.format("SELECT COUNT(*) FROM %s WHERE arrayLength(%s) = 4",
+ getTableName(), BIG_DECIMAL_ARRAY_COLUMN);
+ assertEquals(postQuery(queryBigDecimal).get("resultTable").get("rows").get(0).get(0).asLong(),
+ getCountStarResult());
+
+ String queryBytes = String.format("SELECT COUNT(*) FROM %s WHERE arrayLength(%s) = 4",
+ getTableName(), BYTES_ARRAY_COLUMN);
+ assertEquals(postQuery(queryBytes).get("resultTable").get("rows").get(0).get(0).asLong(),
+ getCountStarResult());
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArraysOverlapBigDecimalAndBytes(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ String queryBigDecimal = String.format("SELECT COUNT(*) FROM %s WHERE ARRAYS_OVERLAP(%s, %s)",
+ getTableName(), BIG_DECIMAL_ARRAY_COLUMN, BIG_DECIMAL_ARRAY_COLUMN);
+ assertEquals(postQuery(queryBigDecimal).get("resultTable").get("rows").get(0).get(0).asLong(),
+ getCountStarResult());
+
+ String queryBytes = String.format("SELECT COUNT(*) FROM %s WHERE ARRAYS_OVERLAP(%s, %s)",
+ getTableName(), BYTES_ARRAY_COLUMN, BYTES_ARRAY_COLUMN);
+ assertEquals(postQuery(queryBytes).get("resultTable").get("rows").get(0).get(0).asLong(),
+ getCountStarResult());
+ }
+
@Test(dataProvider = "useBothQueryEngines")
public void testFilterMvLongArrayFilterValues(boolean useMultiStageQueryEngine)
throws Exception {
@@ -1253,8 +1286,9 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
}
@Override
- public Schema createSchema() {
- return new Schema.SchemaBuilder().setSchemaName(getTableName())
+ public org.apache.pinot.spi.data.Schema createSchema() {
+ return new org.apache.pinot.spi.data.Schema.SchemaBuilder()
+ .setSchemaName(getTableName())
.addSingleValueDimension(BOOLEAN_COLUMN, FieldSpec.DataType.BOOLEAN)
.addSingleValueDimension(BOOLEAN_FROM_INT_COLUMN, FieldSpec.DataType.BOOLEAN)
.addSingleValueDimension(BOOLEAN_FROM_STRING_COLUMN, FieldSpec.DataType.BOOLEAN)
@@ -1270,7 +1304,9 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
.addMultiValueDimension(BOOLEAN_FROM_STRING_ARRAY_COLUMN, FieldSpec.DataType.BOOLEAN)
.addMultiValueDimension(LONG_ARRAY_COLUMN, FieldSpec.DataType.LONG)
.addMultiValueDimension(DOUBLE_ARRAY_COLUMN, FieldSpec.DataType.DOUBLE)
+ .addMultiValueDimension(BIG_DECIMAL_ARRAY_COLUMN, FieldSpec.DataType.BIG_DECIMAL)
.addMultiValueDimension(STRING_ARRAY_COLUMN, FieldSpec.DataType.STRING)
+ .addMultiValueDimension(BYTES_ARRAY_COLUMN, FieldSpec.DataType.BYTES)
.build();
}
@@ -1278,86 +1314,70 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
public List<File> createAvroFiles()
throws Exception {
// create avro schema
- org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+ Schema avroSchema = Schema.createRecord("myRecord", null, null, false);
avroSchema.setFields(List.of(
- new org.apache.avro.Schema.Field(BOOLEAN_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN),
- null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_FROM_INT_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT),
- null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_FROM_STRING_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING),
- null, null),
- new org.apache.avro.Schema.Field(INT_COLUMN, org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT),
- null, null),
- new org.apache.avro.Schema.Field(LONG_COLUMN, org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG),
- null, null),
- new org.apache.avro.Schema.Field(FLOAT_COLUMN, org.apache.avro.Schema.create(org.apache.avro.Schema.Type.FLOAT),
- null, null),
- new org.apache.avro.Schema.Field(DOUBLE_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE),
- null, null),
- new org.apache.avro.Schema.Field(STRING_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING),
- null, null),
- new org.apache.avro.Schema.Field(TIMESTAMP_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG),
- null, null),
- new org.apache.avro.Schema.Field(GROUP_BY_COLUMN,
- org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING),
- null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BOOLEAN)),
- null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_FROM_INT_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT)),
- null, null),
- new org.apache.avro.Schema.Field(BOOLEAN_FROM_STRING_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING)),
- null, null),
- new org.apache.avro.Schema.Field(LONG_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG)),
- null, null),
- new org.apache.avro.Schema.Field(DOUBLE_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE)),
- null, null),
- new org.apache.avro.Schema.Field(STRING_ARRAY_COLUMN,
- org.apache.avro.Schema.createArray(org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING)),
- null, null)
- ));
-
- Cache<Integer, GenericData.Record> recordCache = CacheBuilder.newBuilder().build();
+ new Schema.Field(BOOLEAN_COLUMN, Schema.create(Schema.Type.BOOLEAN), null, null),
+ new Schema.Field(BOOLEAN_FROM_INT_COLUMN, Schema.create(Schema.Type.INT), null, null),
+ new Schema.Field(BOOLEAN_FROM_STRING_COLUMN, Schema.create(Schema.Type.STRING), null, null),
+ new Schema.Field(INT_COLUMN, Schema.create(Schema.Type.INT), null, null),
+ new Schema.Field(LONG_COLUMN, Schema.create(Schema.Type.LONG), null, null),
+ new Schema.Field(FLOAT_COLUMN, Schema.create(Schema.Type.FLOAT), null, null),
+ new Schema.Field(DOUBLE_COLUMN, Schema.create(Schema.Type.DOUBLE), null, null),
+ new Schema.Field(STRING_COLUMN, Schema.create(Schema.Type.STRING), null, null),
+ new Schema.Field(TIMESTAMP_COLUMN, Schema.create(Schema.Type.LONG), null, null),
+ new Schema.Field(GROUP_BY_COLUMN, Schema.create(Schema.Type.STRING), null, null),
+ new Schema.Field(BOOLEAN_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.BOOLEAN)), null, null),
+ new Schema.Field(BOOLEAN_FROM_INT_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.INT)), null, null),
+ new Schema.Field(BOOLEAN_FROM_STRING_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.STRING)), null,
+ null),
+ new Schema.Field(LONG_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.LONG)), null, null),
+ new Schema.Field(DOUBLE_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.DOUBLE)), null, null),
+ new Schema.Field(BIG_DECIMAL_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.STRING)), null, null),
+ new Schema.Field(STRING_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.STRING)), null, null),
+ new Schema.Field(BYTES_ARRAY_COLUMN, Schema.createArray(Schema.create(Schema.Type.BYTES)), null, null))
+ );
+
+ int numRecords = (int) getCountStarResult();
+ int numUniqueRecords = numRecords / 10;
+ List<GenericData.Record> uniqueRecords = new ArrayList<>(numUniqueRecords);
+ for (int i = 0; i < numUniqueRecords; i++) {
+ uniqueRecords.add(createAvroRecord(avroSchema, i));
+ }
try (AvroFilesAndWriters avroFilesAndWriters = createAvroFilesAndWriters(avroSchema)) {
List<DataFileWriter<GenericData.Record>> writers = avroFilesAndWriters.getWriters();
- for (int i = 0; i < getCountStarResult(); i++) {
- // add avro record to file
- int finalI = i;
- writers.get(i % getNumAvroFiles())
- .append(recordCache.get((int) (i % (getCountStarResult() / 10)), () -> {
- // create avro record
- GenericData.Record record = new GenericData.Record(avroSchema);
- record.put(BOOLEAN_COLUMN, finalI % 4 == 0 || finalI % 4 == 1);
- record.put(BOOLEAN_FROM_INT_COLUMN, finalI % 4 == 0 || finalI % 4 == 1 ? 1 : 0);
- record.put(BOOLEAN_FROM_STRING_COLUMN, finalI % 4 == 0 || finalI % 4 == 1 ? "true" : "false");
- record.put(INT_COLUMN, finalI);
- record.put(LONG_COLUMN, finalI);
- record.put(FLOAT_COLUMN, finalI + RANDOM.nextFloat());
- record.put(DOUBLE_COLUMN, finalI + RANDOM.nextDouble());
- record.put(STRING_COLUMN, RandomStringUtils.random(finalI));
- record.put(TIMESTAMP_COLUMN, finalI);
- record.put(GROUP_BY_COLUMN, String.valueOf(finalI % 10));
- record.put(BOOLEAN_ARRAY_COLUMN, List.of(true, true, false, false));
- record.put(BOOLEAN_FROM_INT_ARRAY_COLUMN, List.of(1, 1, 0, 0));
- record.put(BOOLEAN_FROM_STRING_ARRAY_COLUMN, List.of("true", "true", "false", "false"));
- record.put(LONG_ARRAY_COLUMN, List.of(0, 1, 2, 3));
- record.put(DOUBLE_ARRAY_COLUMN, List.of(0.0, 0.1, 0.2, 0.3));
- record.put(STRING_ARRAY_COLUMN, List.of("/api/v1", "/home", "/api/v2", "/metrics"));
- return record;
- }
- ));
+ int numAvroFiles = getNumAvroFiles();
+ for (int i = 0; i < numRecords; i++) {
+ writers.get(i % numAvroFiles).append(uniqueRecords.get(i % numUniqueRecords));
}
return avroFilesAndWriters.getAvroFiles();
}
}
+
+ private GenericData.Record createAvroRecord(Schema schema, int i) {
+ GenericData.Record record = new GenericData.Record(schema);
+ record.put(BOOLEAN_COLUMN, i % 4 == 0 || i % 4 == 1);
+ record.put(BOOLEAN_FROM_INT_COLUMN, i % 4 == 0 || i % 4 == 1 ? 1 : 0);
+ record.put(BOOLEAN_FROM_STRING_COLUMN, i % 4 == 0 || i % 4 == 1 ? "true" : "false");
+ record.put(INT_COLUMN, i);
+ record.put(LONG_COLUMN, i);
+ record.put(FLOAT_COLUMN, i + RANDOM.nextFloat());
+ record.put(DOUBLE_COLUMN, i + RANDOM.nextDouble());
+ record.put(STRING_COLUMN, RandomStringUtils.secure().next(i));
+ record.put(TIMESTAMP_COLUMN, i);
+ record.put(GROUP_BY_COLUMN, String.valueOf(i % 10));
+ record.put(BOOLEAN_ARRAY_COLUMN, List.of(true, true, false, false));
+ record.put(BOOLEAN_FROM_INT_ARRAY_COLUMN, List.of(1, 1, 0, 0));
+ record.put(BOOLEAN_FROM_STRING_ARRAY_COLUMN, List.of("true", "true", "false", "false"));
+ record.put(LONG_ARRAY_COLUMN, List.of(0, 1, 2, 3));
+ record.put(DOUBLE_ARRAY_COLUMN, List.of(0.0, 0.1, 0.2, 0.3));
+ record.put(BIG_DECIMAL_ARRAY_COLUMN, List.of("0.0", "0.1", "0.2", "0.3"));
+ record.put(STRING_ARRAY_COLUMN, List.of("/api/v1", "/home", "/api/v2", "/metrics"));
+ record.put(BYTES_ARRAY_COLUMN, List.of(
+ ByteBuffer.wrap(new byte[]{0x00}),
+ ByteBuffer.wrap(new byte[]{0x01}),
+ ByteBuffer.wrap(new byte[]{0x02}),
+ ByteBuffer.wrap(new byte[]{0x03})
+ ));
+ return record;
+ }
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BigDecimalTypeTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BigDecimalTypeTest.java
new file mode 100644
index 00000000000..4f323528166
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BigDecimalTypeTest.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// End-to-end coverage for BIG_DECIMAL as a dimension, both single-valued and multi-valued. Metric-side BIG_DECIMAL
+/// coverage lives in [SumPrecisionTest] and [ArithmeticFunctionsIntegrationTest]; this test focuses on
+/// ingestion, projection, filtering, aggregation, and cast of BIG_DECIMAL dimensions — in particular, the MV path that
+/// was previously unsupported.
+///
+/// Each BIG_DECIMAL column appears in two variants — dictionary-encoded (default) and raw (no-dictionary) — so the
+/// tests exercise both encodings. The raw variants are configured via [#getNoDictionaryColumns()].
+@Test(suiteName = "CustomClusterIntegrationTest")
+public class BigDecimalTypeTest extends CustomDataQueryClusterIntegrationTest {
+ private static final String DEFAULT_TABLE_NAME = "BigDecimalTypeTest";
+ private static final int NUM_DOCS = 100;
+ private static final int MV_LENGTH = 3;
+
+ private static final String ID_COLUMN = "id";
+ // Dictionary-encoded variants
+ private static final String SV_COLUMN = "bigDecimalSV";
+ private static final String MV_COLUMN = "bigDecimalMV";
+ // Raw (no-dictionary) variants
+ private static final String RAW_SV_COLUMN = "rawBigDecimalSV";
+ private static final String RAW_MV_COLUMN = "rawBigDecimalMV";
+ // Used to exercise CAST(... AS BIG_DECIMAL_ARRAY) and implicit conversion paths.
+ private static final String DOUBLE_MV_COLUMN = "doubleMV";
+
+ private static final String[] SV_COLUMNS = {SV_COLUMN, RAW_SV_COLUMN};
+ private static final String[] MV_COLUMNS = {MV_COLUMN, RAW_MV_COLUMN};
+
+ @Override
+ public String getTableName() {
+ return DEFAULT_TABLE_NAME;
+ }
+
+ @Override
+ protected long getCountStarResult() {
+ return NUM_DOCS;
+ }
+
+ @Override
+ public TableConfig createOfflineTableConfig() {
+ return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
+ .setNoDictionaryColumns(getNoDictionaryColumns()).build();
+ }
+
+ @Override
+ protected List<String> getNoDictionaryColumns() {
+ return List.of(RAW_SV_COLUMN, RAW_MV_COLUMN);
+ }
+
+ @Override
+ public Schema createSchema() {
+ return new Schema.SchemaBuilder()
+ .setSchemaName(getTableName())
+ .addSingleValueDimension(ID_COLUMN, DataType.INT)
+ .addSingleValueDimension(SV_COLUMN, DataType.BIG_DECIMAL)
+ .addSingleValueDimension(RAW_SV_COLUMN, DataType.BIG_DECIMAL)
+ .addMultiValueDimension(MV_COLUMN, DataType.BIG_DECIMAL)
+ .addMultiValueDimension(RAW_MV_COLUMN, DataType.BIG_DECIMAL)
+ .addMultiValueDimension(DOUBLE_MV_COLUMN, DataType.DOUBLE)
+ .build();
+ }
+
+ /**
+ * For doc {@code i}, SV = {@code i.5} and MV = {@code [i.0, i.25, i.5]}. The values are chosen so that min/max and
+ * sum-style aggregations yield exact BigDecimal results without double-precision rounding.
+ */
+ private static BigDecimal svValue(int i) {
+ return new BigDecimal(i + ".5");
+ }
+
+ private static BigDecimal[] mvValues(int i) {
+ return new BigDecimal[]{
+ new BigDecimal(i + ".00"),
+ new BigDecimal(i + ".25"),
+ new BigDecimal(i + ".50")
+ };
+ }
+
+ @Override
+ public List<File> createAvroFiles()
+ throws IOException {
+ // BIG_DECIMAL ingestion goes through string → BigDecimal conversion in the transform pipeline, so we encode
+ // BigDecimal values as Avro STRING / STRING_ARRAY rather than fighting with Avro's logical DECIMAL type.
+ org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+ org.apache.avro.Schema stringSchema = org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING);
+ org.apache.avro.Schema doubleSchema = org.apache.avro.Schema.create(org.apache.avro.Schema.Type.DOUBLE);
+ avroSchema.setFields(List.of(
+ new org.apache.avro.Schema.Field(ID_COLUMN, org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT),
+ null, null),
+ new org.apache.avro.Schema.Field(SV_COLUMN, stringSchema, null, null),
+ new org.apache.avro.Schema.Field(RAW_SV_COLUMN, stringSchema, null, null),
+ new org.apache.avro.Schema.Field(MV_COLUMN, org.apache.avro.Schema.createArray(stringSchema), null, null),
+ new org.apache.avro.Schema.Field(RAW_MV_COLUMN, org.apache.avro.Schema.createArray(stringSchema), null, null),
+ new org.apache.avro.Schema.Field(DOUBLE_MV_COLUMN, org.apache.avro.Schema.createArray(doubleSchema), null, null)
+ ));
+
+ try (AvroFilesAndWriters avroFilesAndWriters = createAvroFilesAndWriters(avroSchema)) {
+ List<DataFileWriter<GenericData.Record>> writers = avroFilesAndWriters.getWriters();
+ for (int i = 0; i < NUM_DOCS; i++) {
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put(ID_COLUMN, i);
+ String svText = svValue(i).toPlainString();
+ record.put(SV_COLUMN, svText);
+ record.put(RAW_SV_COLUMN, svText);
+ List<String> mvStrings = new ArrayList<>(MV_LENGTH);
+ List<Double> mvDoubles = new ArrayList<>(MV_LENGTH);
+ for (BigDecimal v : mvValues(i)) {
+ mvStrings.add(v.toPlainString());
+ mvDoubles.add(v.doubleValue());
+ }
+ record.put(MV_COLUMN, mvStrings);
+ record.put(RAW_MV_COLUMN, mvStrings);
+ record.put(DOUBLE_MV_COLUMN, mvDoubles);
+ writers.get(i % getNumAvroFiles()).append(record);
+ }
+ return avroFilesAndWriters.getAvroFiles();
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testSvProjection(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String svCol : SV_COLUMNS) {
+ String query = String.format("SELECT %s, %s FROM %s WHERE %s = 42 LIMIT 1", ID_COLUMN, svCol, getTableName(),
+ ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.size(), 1);
+ JsonNode row = rows.get(0);
+ assertEquals(row.get(0).asInt(), 42);
+ assertEquals(new BigDecimal(row.get(1).asText()), svValue(42));
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvProjection(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ BigDecimal[] expected = mvValues(7);
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT %s FROM %s WHERE %s = 7 LIMIT 1", mvCol, getTableName(), ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.size(), 1);
+ JsonNode mv = rows.get(0).get(0);
+ assertEquals(mv.size(), MV_LENGTH);
+ // Use compareTo (scale-insensitive) because BIG_DECIMAL columns strip trailing zeros by default; "7.00" comes
+ // back as "7" and "7.50" as "7.5". Stored-vs-expected equivalence is numeric, not textual.
+ for (int i = 0; i < MV_LENGTH; i++) {
+ assertEquals(new BigDecimal(mv.get(i).asText()).compareTo(expected[i]), 0);
+ }
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testSvFilter(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // Match docs where the SV BigDecimal exceeds 10.5, which is i >= 11.
+ for (String svCol : SV_COLUMNS) {
+ String query = String.format("SELECT count(*) FROM %s WHERE %s > 10.5", getTableName(), svCol);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asLong(), NUM_DOCS - 11);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvFilter(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // MV predicate matches a doc if any element of the array satisfies. For doc i, mvValues contain i.0/i.25/i.5, so
+ // the largest element is i.5; filter bigDecimalMV >= 10.5 matches i >= 10. MSE requires ARRAY_TO_MV() to compare
+ // an MV column to a scalar; SSE accepts the direct MV comparison.
+ for (String mvCol : MV_COLUMNS) {
+ String mvExpr = useMultiStageQueryEngine ? String.format("ARRAY_TO_MV(%s)", mvCol) : mvCol;
+ String query = String.format("SELECT count(*) FROM %s WHERE %s >= 10.5", getTableName(), mvExpr);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asLong(), NUM_DOCS - 10);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testSvAggregation(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ BigDecimal expectedSum = BigDecimal.ZERO;
+ for (int i = 0; i < NUM_DOCS; i++) {
+ expectedSum = expectedSum.add(svValue(i));
+ }
+ for (String svCol : SV_COLUMNS) {
+ String query = String.format("SELECT min(%s), max(%s), sumPrecision(%s) FROM %s",
+ svCol, svCol, svCol, getTableName());
+ JsonNode row = postQuery(query).get("resultTable").get("rows").get(0);
+ // Stored values are 0.5, 1.5, ..., 99.5
+ assertEquals(new BigDecimal(row.get(0).asText()), svValue(0));
+ assertEquals(new BigDecimal(row.get(1).asText()), svValue(NUM_DOCS - 1));
+ assertEquals(new BigDecimal(row.get(2).asText()), expectedSum);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvMinMax(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT minMV(%s), maxMV(%s) FROM %s", mvCol, mvCol, getTableName());
+ JsonNode row = postQuery(query).get("resultTable").get("rows").get(0);
+ // minMV picks the smallest element across all (doc, element) pairs => 0.00 (doc 0, first element).
+ // maxMV picks the largest => 99.50 (doc 99, last element).
+ assertEquals(new BigDecimal(row.get(0).asText()).compareTo(new BigDecimal("0.00")), 0);
+ assertEquals(new BigDecimal(row.get(1).asText()).compareTo(new BigDecimal("99.50")), 0);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvCardinality(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // CARDINALITY on an MV column returns the number of elements per row.
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT cardinality(%s) FROM %s WHERE %s = 0 LIMIT 1", mvCol, getTableName(),
+ ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asInt(), MV_LENGTH);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testGroupBySv(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String svCol : SV_COLUMNS) {
+ String query =
+ String.format("SELECT %s, count(*) FROM %s GROUP BY %s ORDER BY %s LIMIT %d", svCol, getTableName(),
+ svCol, svCol, NUM_DOCS);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.size(), NUM_DOCS);
+ // Each SV value is unique, so every group has exactly one row.
+ for (int i = 0; i < NUM_DOCS; i++) {
+ assertEquals(new BigDecimal(rows.get(i).get(0).asText()), svValue(i));
+ assertEquals(rows.get(i).get(1).asLong(), 1L);
+ }
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testCastMvToBigDecimal(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // MSE uses standard Calcite `TYPE ARRAY` syntax; SSE uses Pinot's underscored `BIG_DECIMAL_ARRAY` identifier
+ // understood by CastTransformFunction.
+ String castTargetType = useMultiStageQueryEngine ? "DECIMAL ARRAY" : "BIG_DECIMAL_ARRAY";
+ String query = String.format(
+ "SELECT cardinality(cast(%s as %s)) FROM %s WHERE %s = 0 LIMIT 1",
+ DOUBLE_MV_COLUMN, castTargetType, getTableName(), ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asInt(), MV_LENGTH);
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testSelectStar(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // A plain select-all to verify schema reporting for BIG_DECIMAL / BIG_DECIMAL_ARRAY columns round-trips cleanly.
+ String query = String.format("SELECT * FROM %s WHERE %s = 0 LIMIT 1", getTableName(), ID_COLUMN);
+ JsonNode result = postQuery(query).get("resultTable");
+ JsonNode rows = result.get("rows");
+ assertEquals(rows.size(), 1);
+ assertTrue(result.get("dataSchema").get("columnDataTypes").toString().contains("BIG_DECIMAL"));
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
new file mode 100644
index 00000000000..5e7f2c8c337
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// End-to-end coverage for BYTES as a multi-valued dimension. SV BYTES is covered by [BytesTypeTest]; this test
+/// exercises the MV path: ingestion (Avro `array<bytes>`), raw-MV segment storage, projection via
+/// [org.apache.pinot.core.common.RowBasedBlockValueFetcher], DataTable ser/de, and broker-side extraction.
+///
+/// The MV BYTES column appears in two variants — dictionary-encoded (default) and raw (no-dictionary) — so the
+/// tests exercise both encodings. The raw variant is configured via [#getNoDictionaryColumns()].
+@Test(suiteName = "CustomClusterIntegrationTest")
+public class BytesMvTypeTest extends CustomDataQueryClusterIntegrationTest {
+ private static final String DEFAULT_TABLE_NAME = "BytesMvTypeTest";
+ private static final int NUM_DOCS = 100;
+ private static final int MV_LENGTH = 3;
+
+ private static final String ID_COLUMN = "id";
+ // Dictionary-encoded variant (default)
+ private static final String BYTES_MV_COLUMN = "bytesMV";
+ // Raw (no-dictionary) variant
+ private static final String RAW_BYTES_MV_COLUMN = "rawBytesMV";
+
+ private static final String[] MV_COLUMNS = {BYTES_MV_COLUMN, RAW_BYTES_MV_COLUMN};
+
+ @Override
+ public String getTableName() {
+ return DEFAULT_TABLE_NAME;
+ }
+
+ @Override
+ protected long getCountStarResult() {
+ return NUM_DOCS;
+ }
+
+ @Override
+ public TableConfig createOfflineTableConfig() {
+ return new TableConfigBuilder(TableType.OFFLINE).setTableName(getTableName())
+ .setNoDictionaryColumns(getNoDictionaryColumns()).build();
+ }
+
+ @Override
+ protected List<String> getNoDictionaryColumns() {
+ return List.of(RAW_BYTES_MV_COLUMN);
+ }
+
+ @Override
+ public Schema createSchema() {
+ return new Schema.SchemaBuilder()
+ .setSchemaName(getTableName())
+ .addSingleValueDimension(ID_COLUMN, DataType.INT)
+ .addMultiValueDimension(BYTES_MV_COLUMN, DataType.BYTES)
+ .addMultiValueDimension(RAW_BYTES_MV_COLUMN, DataType.BYTES)
+ .build();
+ }
+
+ /// For doc `i`, MV elements are `[0xNN, 0xMM, 0xPP]` where each element encodes `i` in a different
+ /// way. Chosen so that both content and order are deterministic and easy to assert.
+ private static byte[][] mvValues(int i) {
+ return new byte[][]{
+ new byte[]{(byte) (i & 0xFF)},
+ new byte[]{(byte) (i & 0xFF), (byte) ((i + 1) & 0xFF)},
+ new byte[]{(byte) (i & 0xFF), (byte) ((i + 2) & 0xFF), (byte) ((i + 3) & 0xFF)}
+ };
+ }
+
+ @Override
+ public List<File> createAvroFiles()
+ throws IOException {
+ org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false);
+ org.apache.avro.Schema bytesSchema = org.apache.avro.Schema.create(org.apache.avro.Schema.Type.BYTES);
+ avroSchema.setFields(List.of(
+ new org.apache.avro.Schema.Field(ID_COLUMN, org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT),
+ null,
+ null),
+ new org.apache.avro.Schema.Field(BYTES_MV_COLUMN, org.apache.avro.Schema.createArray(bytesSchema), null, null),
+ new org.apache.avro.Schema.Field(RAW_BYTES_MV_COLUMN, org.apache.avro.Schema.createArray(bytesSchema), null,
+ null)
+ ));
+
+ try (AvroFilesAndWriters avroFilesAndWriters = createAvroFilesAndWriters(avroSchema)) {
+ List<DataFileWriter<GenericData.Record>> writers = avroFilesAndWriters.getWriters();
+ for (int i = 0; i < NUM_DOCS; i++) {
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put(ID_COLUMN, i);
+ // Each call wraps fresh byte[]s so the two columns don't share ByteBuffer state across writers.
+ record.put(BYTES_MV_COLUMN, wrapBuffers(mvValues(i)));
+ record.put(RAW_BYTES_MV_COLUMN, wrapBuffers(mvValues(i)));
+ writers.get(i % getNumAvroFiles()).append(record);
+ }
+ return avroFilesAndWriters.getAvroFiles();
+ }
+ }
+
+ private static List<ByteBuffer> wrapBuffers(byte[][] elements) {
+ List<ByteBuffer> buffers = new ArrayList<>(elements.length);
+ for (byte[] element : elements) {
+ buffers.add(ByteBuffer.wrap(element));
+ }
+ return buffers;
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvProjection(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ byte[][] expected = mvValues(7);
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT %s FROM %s WHERE %s = 7 LIMIT 1", mvCol, getTableName(), ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.size(), 1);
+ JsonNode mv = rows.get(0).get(0);
+ assertEquals(mv.size(), MV_LENGTH);
+ for (int i = 0; i < MV_LENGTH; i++) {
+ // Pinot returns MV BYTES elements as hex-encoded strings in the JSON response.
+ assertEquals(mv.get(i).asText(), Hex.encodeHexString(expected[i]));
+ }
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testMvCardinality(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT cardinality(%s) FROM %s WHERE %s = 0 LIMIT 1", mvCol, getTableName(),
+ ID_COLUMN);
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asInt(), MV_LENGTH);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testSelectWithMvColumn(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ // SELECT id, mv returns the MV BYTES column alongside id — exercises the full projection + selection + broker
+ // extract path for MV BYTES.
+ for (String mvCol : MV_COLUMNS) {
+ String query = String.format("SELECT %s, %s FROM %s WHERE %s = 0 LIMIT 1", ID_COLUMN, mvCol,
+ getTableName(), ID_COLUMN);
+ JsonNode result = postQuery(query).get("resultTable");
+ JsonNode rows = result.get("rows");
+ assertEquals(rows.size(), 1);
+ assertEquals(rows.get(0).get(0).asInt(), 0);
+ assertEquals(rows.get(0).get(1).size(), MV_LENGTH);
+ assertTrue(result.get("dataSchema").get("columnDataTypes").toString().contains("BYTES_ARRAY"));
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testCountStar(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ String query = String.format("SELECT count(*) FROM %s", getTableName());
+ JsonNode rows = postQuery(query).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asLong(), NUM_DOCS);
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesTypeTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesTypeTest.java
index 1c1d54ccb91..ff392d5a691 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesTypeTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesTypeTest.java
@@ -37,9 +37,8 @@ import org.testng.annotations.Test;
@Test(suiteName = "CustomClusterIntegrationTest")
public class BytesTypeTest extends CustomDataQueryClusterIntegrationTest {
-
- protected static final String DEFAULT_TABLE_NAME = "BytesTypeTest";
- private static final String FIXED_HEX_STRIING_VALUE = "968a3c6a5eeb42168bae0e895034a26f";
+ private static final String DEFAULT_TABLE_NAME = "BytesTypeTest";
+ private static final String FIXED_HEX_STRING_VALUE = "968a3c6a5eeb42168bae0e895034a26f";
private static final int NUM_TOTAL_DOCS = 1000;
private static final String HEX_STR = "hexStr";
@@ -146,8 +145,8 @@ public class BytesTypeTest extends CustomDataQueryClusterIntegrationTest {
byte[] randomBytes = newRandomBytes();
record.put(RANDOM_STR, new String(randomBytes));
record.put(RANDOM_BYTES, ByteBuffer.wrap(randomBytes));
- record.put(FIXED_STRING, FIXED_HEX_STRIING_VALUE);
- record.put(FIXED_BYTES, ByteBuffer.wrap(DataTypeConversionFunctions.hexToBytes(FIXED_HEX_STRIING_VALUE)));
+ record.put(FIXED_STRING, FIXED_HEX_STRING_VALUE);
+ record.put(FIXED_BYTES, ByteBuffer.wrap(DataTypeConversionFunctions.hexToBytes(FIXED_HEX_STRING_VALUE)));
writers.get(i % getNumAvroFiles()).append(record);
}
return avroFilesAndWriters.getAvroFiles();
@@ -281,7 +280,7 @@ public class BytesTypeTest extends CustomDataQueryClusterIntegrationTest {
// String predicate
String query =
- String.format("Select count(*) from %s WHERE %s = '%s'", getTableName(), FIXED_STRING, FIXED_HEX_STRIING_VALUE);
+ String.format("Select count(*) from %s WHERE %s = '%s'", getTableName(), FIXED_STRING, FIXED_HEX_STRING_VALUE);
JsonNode pinotResponse = postQuery(query);
JsonNode rows = pinotResponse.get("resultTable").get("rows");
for (int i = 0; i < rows.size(); i++) {
@@ -291,7 +290,7 @@ public class BytesTypeTest extends CustomDataQueryClusterIntegrationTest {
// Bytes predicate, convert literal string to bytes
query =
String.format("Select count(*) from %s WHERE %s = hexToBytes('%s')", getTableName(), FIXED_BYTES,
- FIXED_HEX_STRIING_VALUE);
+ FIXED_HEX_STRING_VALUE);
pinotResponse = postQuery(query);
rows = pinotResponse.get("resultTable").get("rows");
for (int i = 0; i < rows.size(); i++) {
@@ -301,7 +300,7 @@ public class BytesTypeTest extends CustomDataQueryClusterIntegrationTest {
// Bytes predicate, convert column to hex string to compare with a literal string
query =
String.format("Select count(*) from %s WHERE bytesToHex(%s) = '%s'", getTableName(), FIXED_BYTES,
- FIXED_HEX_STRIING_VALUE);
+ FIXED_HEX_STRING_VALUE);
pinotResponse = postQuery(query);
rows = pinotResponse.get("resultTable").get("rows");
for (int i = 0; i < rows.size(); i++) {
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
index 9a1834d1fcb..831472e28d9 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScanDocIdIterators.java
@@ -219,6 +219,11 @@ public class BenchmarkScanDocIdIterators {
return false;
}
+ @Override
+ public boolean applyMV(BigDecimal[] values, int length) {
+ return false;
+ }
+
@Override
public boolean applyMV(double[] values, int length) {
return false;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/BenchmarkDistinctCountHLLThreshold.java b/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/BenchmarkDistinctCountHLLThreshold.java
index 1849dc8ec32..d326c1300dc 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/BenchmarkDistinctCountHLLThreshold.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/BenchmarkDistinctCountHLLThreshold.java
@@ -334,6 +334,11 @@ public class BenchmarkDistinctCountHLLThreshold {
throw new UnsupportedOperationException();
}
+ @Override
+ public BigDecimal[][] getBigDecimalValuesMV() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String[][] getStringValuesMV() {
throw new UnsupportedOperationException();
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
index 11cf2204dd8..aaeda232871 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
@@ -941,7 +941,8 @@ public final class RelToPlanNodeConverter {
boolean isArray = (sqlTypeName == SqlTypeName.ARRAY);
if (isArray) {
assert relDataType.getComponentType() != null;
- sqlTypeName = relDataType.getComponentType().getSqlTypeName();
+ relDataType = relDataType.getComponentType();
+ sqlTypeName = relDataType.getSqlTypeName();
}
switch (sqlTypeName) {
case BOOLEAN:
@@ -1005,6 +1006,8 @@ public final class RelToPlanNodeConverter {
} else if (precision <= 38) {
return isArray ? ColumnDataType.LONG_ARRAY : ColumnDataType.LONG;
} else {
+ // TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after `1.6.0` release
+ // BIG_DECIMAL_ARRAY support is added in `1.6.0`
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.BIG_DECIMAL;
}
} else {
@@ -1013,6 +1016,8 @@ public final class RelToPlanNodeConverter {
if (precision <= 30) {
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.DOUBLE;
} else {
+ // TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after `1.6.0` release
+ // BIG_DECIMAL_ARRAY support is added in `1.6.0`
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.BIG_DECIMAL;
}
}
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
index a3838af71ac..6e16892a34f 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java
@@ -136,7 +136,6 @@ public class RexExpressionUtils {
TimestampString tsString = TimestampString.fromMillisSinceEpoch((long) value);
return rexBuilder.makeTimestampLiteral(tsString, 1);
}
- case JSON:
case STRING: {
assert value != null;
return rexBuilder.makeLiteral((String) value);
@@ -148,16 +147,6 @@ public class RexExpressionUtils {
ByteString byteString = new ByteString(bytes);
return rexBuilder.makeBinaryLiteral(byteString);
}
- case BOOLEAN_ARRAY:
- case BYTES_ARRAY:
- case DOUBLE_ARRAY:
- case FLOAT_ARRAY:
- case INT_ARRAY:
- case LONG_ARRAY:
- case STRING_ARRAY:
- case TIMESTAMP_ARRAY:
- case OBJECT:
- case UNKNOWN:
default:
throw new IllegalStateException("Unsupported ColumnDataType: " + literal.getDataType());
}
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
index 8a79f61fe29..7217d4676ad 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/v2/PRelToPlanNodeConverter.java
@@ -298,7 +298,8 @@ public class PRelToPlanNodeConverter {
boolean isArray = (sqlTypeName == SqlTypeName.ARRAY);
if (isArray) {
assert relDataType.getComponentType() != null;
- sqlTypeName = relDataType.getComponentType().getSqlTypeName();
+ relDataType = relDataType.getComponentType();
+ sqlTypeName = relDataType.getSqlTypeName();
}
switch (sqlTypeName) {
case BOOLEAN:
@@ -362,6 +363,8 @@ public class PRelToPlanNodeConverter {
} else if (precision <= 38) {
return isArray ? ColumnDataType.LONG_ARRAY : ColumnDataType.LONG;
} else {
+ // TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after `1.6.0` release
+ // BIG_DECIMAL_ARRAY support is added in `1.6.0`
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.BIG_DECIMAL;
}
} else {
@@ -370,6 +373,8 @@ public class PRelToPlanNodeConverter {
if (precision <= 30) {
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.DOUBLE;
} else {
+ // TODO: Modify it to return ColumnDataType.BIG_DECIMAL_ARRAY after `1.6.0` release
+ // BIG_DECIMAL_ARRAY support is added in `1.6.0`
return isArray ? ColumnDataType.DOUBLE_ARRAY : ColumnDataType.BIG_DECIMAL;
}
}
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java
index cca846958e3..68878309281 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.query.planner.serde;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import org.apache.pinot.common.proto.Expressions;
@@ -125,6 +126,15 @@ public class ProtoExpressionToRexExpression {
}
return new RexExpression.Literal(dataType, values);
}
+ case BIG_DECIMAL_ARRAY: {
+ Expressions.BytesArray bytesArray = literal.getBytesArray();
+ int numValues = bytesArray.getValuesCount();
+ BigDecimal[] values = new BigDecimal[numValues];
+ for (int i = 0; i < numValues; i++) {
+ values[i] = BigDecimalUtils.deserialize(bytesArray.getValues(i).toByteArray());
+ }
+ return new RexExpression.Literal(dataType, values);
+ }
case STRING_ARRAY: {
Expressions.StringArray stringArray = literal.getStringArray();
int numValues = stringArray.getValuesCount();
@@ -136,6 +146,15 @@ public class ProtoExpressionToRexExpression {
}
return new RexExpression.Literal(dataType, values);
}
+ case BYTES_ARRAY: {
+ Expressions.BytesArray bytesArray = literal.getBytesArray();
+ int numValues = bytesArray.getValuesCount();
+ ByteArray[] values = new ByteArray[numValues];
+ for (int i = 0; i < numValues; i++) {
+ values[i] = new ByteArray(bytesArray.getValues(i).toByteArray());
+ }
+ return new RexExpression.Literal(dataType, values);
+ }
default:
throw new IllegalStateException("Unsupported ColumnDataType: " + dataType);
}
@@ -171,6 +190,8 @@ public class ProtoExpressionToRexExpression {
return ColumnDataType.FLOAT_ARRAY;
case DOUBLE_ARRAY:
return ColumnDataType.DOUBLE_ARRAY;
+ case BIG_DECIMAL_ARRAY:
+ return ColumnDataType.BIG_DECIMAL_ARRAY;
case BOOLEAN_ARRAY:
return ColumnDataType.BOOLEAN_ARRAY;
case TIMESTAMP_ARRAY:
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java
index 072256fe01c..09303fde0eb 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java
@@ -119,10 +119,28 @@ public class RexExpressionToProtoExpression {
literalBuilder.setDoubleArray(
Expressions.DoubleArray.newBuilder().addAllValues(DoubleArrayList.wrap((double[]) value)).build());
break;
+ case BIG_DECIMAL_ARRAY: {
+ BigDecimal[] bigDecimalArray = (BigDecimal[]) value;
+ Expressions.BytesArray.Builder builder = Expressions.BytesArray.newBuilder();
+ for (BigDecimal bigDecimal : bigDecimalArray) {
+ builder.addValues(ByteString.copyFrom(BigDecimalUtils.serialize(bigDecimal)));
+ }
+ literalBuilder.setBytesArray(builder.build());
+ break;
+ }
case STRING_ARRAY:
literalBuilder.setStringArray(
Expressions.StringArray.newBuilder().addAllValues(Arrays.asList((String[]) value)).build());
break;
+ case BYTES_ARRAY: {
+ ByteArray[] bytesArray = (ByteArray[]) value;
+ Expressions.BytesArray.Builder builder = Expressions.BytesArray.newBuilder();
+ for (ByteArray byteArray : bytesArray) {
+ builder.addValues(ByteString.copyFrom(byteArray.getBytes()));
+ }
+ literalBuilder.setBytesArray(builder.build());
+ break;
+ }
default:
throw new IllegalStateException("Unsupported ColumnDataType: " + dataType);
}
@@ -162,6 +180,8 @@ public class RexExpressionToProtoExpression {
return Expressions.ColumnDataType.FLOAT_ARRAY;
case DOUBLE_ARRAY:
return Expressions.ColumnDataType.DOUBLE_ARRAY;
+ case BIG_DECIMAL_ARRAY:
+ return Expressions.ColumnDataType.BIG_DECIMAL_ARRAY;
case BOOLEAN_ARRAY:
return Expressions.ColumnDataType.BOOLEAN_ARRAY;
case TIMESTAMP_ARRAY:
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/TypeUtils.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/TypeUtils.java
index 336c733d56d..1203f82bc32 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/TypeUtils.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/utils/TypeUtils.java
@@ -26,6 +26,7 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.math.BigDecimal;
import org.apache.pinot.common.utils.ArrayListUtils;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.spi.utils.ByteArray;
public class TypeUtils {
@@ -52,49 +53,89 @@ public class TypeUtils {
// For AggregationFunctions that return serialized custom object, e.g. DistinctCountRawHLLAggregationFunction
case STRING:
return value.toString();
+ case BYTES:
+ assert value instanceof ByteArray;
+ return value;
case INT_ARRAY:
if (value instanceof IntArrayList) {
// For ArrayAggregationFunction
return ArrayListUtils.toIntArray((IntArrayList) value);
- } else {
- return value;
}
+ assert value instanceof int[];
+ return value;
case LONG_ARRAY:
if (value instanceof LongArrayList) {
// For FunnelCountAggregationFunction and ArrayAggregationFunction
return ArrayListUtils.toLongArray((LongArrayList) value);
- } else {
- return value;
}
+ assert value instanceof long[];
+ return value;
case FLOAT_ARRAY:
if (value instanceof FloatArrayList) {
// For ArrayAggregationFunction
return ArrayListUtils.toFloatArray((FloatArrayList) value);
- } else if (value instanceof double[]) {
+ }
+ if (value instanceof double[]) {
// This is due to for parsing array literal value like [0.1, 0.2, 0.3].
// The parsed value is stored as double[] in java, however the calcite type is FLOAT_ARRAY.
- float[] floatArray = new float[((double[]) value).length];
+ double[] doubleArray = (double[]) value;
+ float[] floatArray = new float[doubleArray.length];
for (int i = 0; i < floatArray.length; i++) {
- floatArray[i] = (float) ((double[]) value)[i];
+ floatArray[i] = (float) doubleArray[i];
}
return floatArray;
- } else {
- return value;
}
+ assert value instanceof float[];
+ return value;
case DOUBLE_ARRAY:
if (value instanceof DoubleArrayList) {
// For HistogramAggregationFunction and ArrayAggregationFunction
return ArrayListUtils.toDoubleArray((DoubleArrayList) value);
- } else {
- return value;
}
+ if (value instanceof BigDecimal[]) {
+ // MV BIG_DECIMAL columns are reported as DOUBLE_ARRAY at the MSE boundary for backward compatibility
+ // (see RelToPlanNodeConverter.resolveDecimal TODO); downcast to double[] here. Precision loss is
+ // accepted until the TODO is cleared in a future release.
+ BigDecimal[] bigDecimalArray = (BigDecimal[]) value;
+ double[] doubleArray = new double[bigDecimalArray.length];
+ for (int i = 0; i < bigDecimalArray.length; i++) {
+ doubleArray[i] = bigDecimalArray[i].doubleValue();
+ }
+ return doubleArray;
+ }
+ if (value instanceof ObjectArrayList) {
+ // ARRAY_AGG on BIG_DECIMAL produces ObjectArrayList<BigDecimal>; same backward-compat downcast.
+ ObjectArrayList<?> list = (ObjectArrayList<?>) value;
+ int size = list.size();
+ double[] doubleArray = new double[size];
+ for (int i = 0; i < size; i++) {
+ doubleArray[i] = ((BigDecimal) list.get(i)).doubleValue();
+ }
+ return doubleArray;
+ }
+ assert value instanceof double[];
+ return value;
+ case BIG_DECIMAL_ARRAY:
+ if (value instanceof ObjectArrayList) {
+ // For ArrayAggregationFunction
+ return ArrayListUtils.toBigDecimalArray((ObjectArrayList<BigDecimal>) value);
+ }
+ assert value instanceof BigDecimal[];
+ return value;
case STRING_ARRAY:
if (value instanceof ObjectArrayList) {
// For ArrayAggregationFunction
return ArrayListUtils.toStringArray((ObjectArrayList<String>) value);
- } else {
- return value;
}
+ assert value instanceof String[];
+ return value;
+ case BYTES_ARRAY:
+ if (value instanceof ObjectArrayList) {
+ // For ArrayAggregationFunction
+ return ArrayListUtils.toBytesArray((ObjectArrayList<ByteArray>) value);
+ }
+ assert value instanceof ByteArray[];
+ return value;
// TODO: Add more conversions
default:
return value;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
index 0a6bd479256..f04481dc04e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
@@ -118,6 +118,11 @@ public class VarByteChunkForwardIndexWriter extends BaseChunkForwardIndexWriter
putBytes(ArraySerDeUtils.serializeDoubleArrayWithLength(values));
}
+ @Override
+ public void putBigDecimalMV(BigDecimal[] values) {
+ putBytes(ArraySerDeUtils.serializeBigDecimalArray(values));
+ }
+
@Override
public void putStringMV(String[] values) {
putBytes(ArraySerDeUtils.serializeStringArray(values));
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
index ec18e9cb764..c6b30c03848 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
@@ -171,6 +171,11 @@ public class VarByteChunkForwardIndexWriterV4 implements VarByteChunkWriter {
putBytes(ArraySerDeUtils.serializeDoubleArrayWithLength(values));
}
+ @Override
+ public void putBigDecimalMV(BigDecimal[] values) {
+ putBytes(ArraySerDeUtils.serializeBigDecimalArray(values));
+ }
+
@Override
public void putStringMV(String[] values) {
putBytes(ArraySerDeUtils.serializeStringArray(values));
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
index 82fbd2b4691..80739ca63d5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
@@ -37,6 +37,8 @@ public interface VarByteChunkWriter extends Closeable {
void putDoubleMV(double[] values);
+ void putBigDecimalMV(BigDecimal[] values);
+
void putStringMV(String[] values);
void putBytesMV(byte[][] values);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
index b45d7f8741f..40d99fc065a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
@@ -261,8 +261,8 @@ public class SegmentDictionaryCreator implements IndexCreator {
return _floatValueToIndexMap;
case DOUBLE:
return _doubleValueToIndexMap;
- case STRING:
case BIG_DECIMAL:
+ case STRING:
case BYTES:
return _objectValueToIndexMap;
default:
@@ -314,8 +314,8 @@ public class SegmentDictionaryCreator implements IndexCreator {
return _floatValueToIndexMap.get((float) value);
case DOUBLE:
return _doubleValueToIndexMap.get((double) value);
- case STRING:
case BIG_DECIMAL:
+ case STRING:
return _objectValueToIndexMap.getInt(value);
case BYTES:
return _objectValueToIndexMap.getInt(new ByteArray((byte[]) value));
@@ -398,6 +398,14 @@ public class SegmentDictionaryCreator implements IndexCreator {
return indexes;
}
+ public int[] indexOfMV(BigDecimal[] values) {
+ int[] indexes = new int[values.length];
+ for (int i = 0; i < values.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(values[i]);
+ }
+ return indexes;
+ }
+
public int[] indexOfMV(String[] values) {
int[] indexes = new int[values.length];
for (int i = 0; i < values.length; i++) {
@@ -439,6 +447,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
indexes[i] = _doubleValueToIndexMap.get((double) multiValues[i]);
}
break;
+ case BIG_DECIMAL:
case STRING:
for (int i = 0; i < multiValues.length; i++) {
indexes[i] = _objectValueToIndexMap.getInt(multiValues[i]);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
index ff0fc2bc704..01e345d6068 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
@@ -21,6 +21,7 @@ package org.apache.pinot.segment.local.segment.creator.impl.fwd;
import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
+import java.math.BigDecimal;
import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkForwardIndexWriter;
import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkForwardIndexWriterV4;
import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkForwardIndexWriterV6;
@@ -112,6 +113,11 @@ public class MultiValueVarByteRawIndexCreator implements ForwardIndexCreator {
return _valueType;
}
+ @Override
+ public void putBigDecimalMV(final BigDecimal[] values) {
+ _indexWriter.putBigDecimalMV(values);
+ }
+
@Override
public void putStringMV(final String[] values) {
_indexWriter.putStringMV(values);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
index c84a05e24b0..68f79f0be0f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
@@ -32,6 +32,7 @@ public class BigDecimalColumnPreIndexStatsCollector extends AbstractColumnStatis
private ObjectOpenHashSet<BigDecimal> _values = new ObjectOpenHashSet<>(INITIAL_HASH_SET_SIZE);
private int _minLength = Integer.MAX_VALUE;
private int _maxLength = 0;
+ private int _maxRowLength = 0;
private BigDecimal[] _sortedValues;
private boolean _sealed = false;
@@ -44,7 +45,20 @@ public class BigDecimalColumnPreIndexStatsCollector extends AbstractColumnStatis
assert !_sealed;
if (entry instanceof Object[]) {
- throw new UnsupportedOperationException();
+ Object[] values = (Object[]) entry;
+ int rowLength = 0;
+ for (Object obj : values) {
+ BigDecimal value = (BigDecimal) obj;
+ int length = BigDecimalUtils.byteSize(value);
+ if (_values.add(value)) {
+ _minLength = Math.min(_minLength, length);
+ _maxLength = Math.max(_maxLength, length);
+ }
+ rowLength += length;
+ }
+ _maxNumberOfMultiValues = Math.max(_maxNumberOfMultiValues, values.length);
+ _maxRowLength = Math.max(_maxRowLength, rowLength);
+ updateTotalNumberOfEntries(values);
} else {
BigDecimal value = (BigDecimal) entry;
int length = BigDecimalUtils.byteSize(value);
@@ -55,6 +69,7 @@ public class BigDecimalColumnPreIndexStatsCollector extends AbstractColumnStatis
}
_minLength = Math.min(_minLength, length);
_maxLength = Math.max(_maxLength, length);
+ _maxRowLength = _maxLength;
}
_totalNumberOfEntries++;
}
@@ -96,7 +111,7 @@ public class BigDecimalColumnPreIndexStatsCollector extends AbstractColumnStatis
@Override
public int getMaxRowLengthInBytes() {
- return _maxLength;
+ return _maxRowLength;
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
index 6084c77b4ee..97ca0220d2a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
@@ -149,6 +149,7 @@ public class ForwardIndexCreatorFactory {
return new MultiValueFixedByteRawIndexCreator(indexDir, compressionType, column, numTotalDocs, storedType,
maxNumberOfMultiValueElements, deriveNumDocsPerChunk, writerVersion, targetMaxChunkSize,
targetDocsPerChunk);
+ case BIG_DECIMAL:
case STRING:
case BYTES:
return new MultiValueVarByteRawIndexCreator(indexDir, compressionType, column, numTotalDocs, storedType,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
index 7095d2740a3..2473399610f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
@@ -616,6 +616,18 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
break;
}
+ case BIG_DECIMAL: {
+ for (int i = 0; i < numDocs; i++) {
+ if (isSVColumn) {
+ BigDecimal val = reader.getBigDecimal(i, readerContext);
+ creator.putBigDecimal(val);
+ } else {
+ BigDecimal[] bigDecimals = reader.getBigDecimalMV(i, readerContext);
+ creator.putBigDecimalMV(bigDecimals);
+ }
+ }
+ break;
+ }
case STRING: {
for (int i = 0; i < numDocs; i++) {
if (isSVColumn) {
@@ -640,14 +652,6 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
break;
}
- case BIG_DECIMAL: {
- Preconditions.checkState(isSVColumn, "BigDecimal is not supported for MV columns");
- for (int i = 0; i < numDocs; i++) {
- BigDecimal val = reader.getBigDecimal(i, readerContext);
- creator.putBigDecimal(val);
- }
- break;
- }
case MAP: {
for (int i = 0; i < numDocs; i++) {
if (isSVColumn) {
@@ -732,17 +736,17 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
break;
}
- case BYTES: {
+ case BIG_DECIMAL: {
for (int i = 0; i < numDocs; i++) {
if (isSVColumn) {
int dictId = reader.getDictId(i, readerContext);
- byte[] val = dictionaryReader.getBytesValue(dictId);
- creator.putBytes(val);
+ BigDecimal val = dictionaryReader.getBigDecimalValue(dictId);
+ creator.putBigDecimal(val);
} else {
int[] dictIds = reader.getDictIdMV(i, readerContext);
- byte[][] bytes = new byte[dictIds.length][];
- dictionaryReader.readBytesValues(dictIds, dictIds.length, bytes);
- creator.putBytesMV(bytes);
+ BigDecimal[] bigDecimals = new BigDecimal[dictIds.length];
+ dictionaryReader.readBigDecimalValues(dictIds, dictIds.length, bigDecimals);
+ creator.putBigDecimalMV(bigDecimals);
}
}
break;
@@ -762,12 +766,18 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
break;
}
- case BIG_DECIMAL: {
- Preconditions.checkState(isSVColumn, "BigDecimal is not supported for MV columns");
+ case BYTES: {
for (int i = 0; i < numDocs; i++) {
- int dictId = reader.getDictId(i, readerContext);
- BigDecimal val = dictionaryReader.getBigDecimalValue(dictId);
- creator.putBigDecimal(val);
+ if (isSVColumn) {
+ int dictId = reader.getDictId(i, readerContext);
+ byte[] val = dictionaryReader.getBytesValue(dictId);
+ creator.putBytes(val);
+ } else {
+ int[] dictIds = reader.getDictIdMV(i, readerContext);
+ byte[][] bytes = new byte[dictIds.length][];
+ dictionaryReader.readBytesValues(dictIds, dictIds.length, bytes);
+ creator.putBytesMV(bytes);
+ }
}
break;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index 4d45240dd62..3163f8c34fe 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -21,6 +21,7 @@ package org.apache.pinot.segment.local.segment.index.loader;
import com.google.common.base.Preconditions;
import java.io.File;
import java.io.IOException;
+import java.math.BigDecimal;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
@@ -491,6 +492,25 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
}
break;
+ case BIG_DECIMAL:
+ if (_singleValue) {
+ for (int docId = 0; docId < _numDocs; docId++) {
+ creator.putBigDecimal(dictionary.getBigDecimalValue(getInt(_forwardIndexValueBuffer, docId)));
+ }
+ } else {
+ int startIdx = 0;
+ for (int docId = 0; docId < _numDocs; docId++) {
+ int endIdx = getInt(_forwardIndexLengthBuffer, docId);
+ BigDecimal[] values = new BigDecimal[endIdx - startIdx];
+ int valuesIdx = 0;
+ for (int i = startIdx; i < endIdx; i++) {
+ values[valuesIdx++] = dictionary.getBigDecimalValue(getInt(_forwardIndexValueBuffer, i));
+ }
+ creator.putBigDecimalMV(values);
+ startIdx = endIdx;
+ }
+ }
+ break;
case STRING:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
@@ -529,12 +549,6 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
}
break;
- case BIG_DECIMAL:
- Preconditions.checkState(_singleValue, "BIG_DECIMAL type not supported for multi-value columns");
- for (int docId = 0; docId < _numDocs; docId++) {
- creator.putBigDecimal(dictionary.getBigDecimalValue(getInt(_forwardIndexValueBuffer, docId)));
- }
- break;
default:
throw new IllegalStateException("Invalid type" + creator.getValueType() + " cannot create forward index");
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
index 6c9d70c1f81..28c47703f7a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/columnminmaxvalue/ColumnMinMaxValueGenerator.java
@@ -300,16 +300,29 @@ public class ColumnMinMaxValueGenerator {
break;
}
case BIG_DECIMAL: {
- Preconditions.checkState(isSingleValue, "Unsupported multi-value BIG_DECIMAL column: %s", columnName);
BigDecimal min = null;
BigDecimal max = null;
- for (int docId = 0; docId < numDocs; docId++) {
- BigDecimal value = rawIndexReader.getBigDecimal(docId, readerContext);
- if (min == null || min.compareTo(value) > 0) {
- min = value;
+ if (isSingleValue) {
+ for (int docId = 0; docId < numDocs; docId++) {
+ BigDecimal value = rawIndexReader.getBigDecimal(docId, readerContext);
+ if (min == null || min.compareTo(value) > 0) {
+ min = value;
+ }
+ if (max == null || max.compareTo(value) < 0) {
+ max = value;
+ }
}
- if (max == null || max.compareTo(value) < 0) {
- max = value;
+ } else {
+ for (int docId = 0; docId < numDocs; docId++) {
+ BigDecimal[] values = rawIndexReader.getBigDecimalMV(docId, readerContext);
+ for (BigDecimal value : values) {
+ if (min == null || min.compareTo(value) > 0) {
+ min = value;
+ }
+ if (max == null || max.compareTo(value) < 0) {
+ max = value;
+ }
+ }
}
}
minValue = min;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 46a99dd7aaf..8a0951f5575 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -41,6 +41,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.fwd.SingleValueSorted
import org.apache.pinot.segment.local.segment.creator.impl.inv.OffHeapBitmapInvertedIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
import org.apache.pinot.segment.local.segment.creator.impl.stats.AbstractColumnStatisticsCollector;
+import org.apache.pinot.segment.local.segment.creator.impl.stats.BigDecimalColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.creator.impl.stats.BytesColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.creator.impl.stats.DoubleColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.creator.impl.stats.FloatColumnPreIndexStatsCollector;
@@ -689,17 +690,11 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
case BIG_DECIMAL: {
for (int i = 0; i < numDocs; i++) {
- Preconditions.checkState(isSingleValue, "MV BIG_DECIMAL is not supported");
-
- // Skip type conversion if output value is already the required type. If outputValueType is null, that
- // means the transform function returned null for all docs and in that case outputValue will be the
- // default null value for the field type
- if (outputValueType != null && !(outputValues[i] instanceof BigDecimal)) {
- outputValues[i] = outputValueType.toBigDecimal(outputValues[i]);
- }
+ outputValues[i] = getBigDecimalOutputValue(outputValues[i], isSingleValue, outputValueType,
+ (BigDecimal) fieldSpec.getDefaultNullValue());
}
statsCollector = !useNoDictColumnStatsCollector
- ? new DoubleColumnPreIndexStatsCollector(column, statsCollectorConfig)
+ ? new BigDecimalColumnPreIndexStatsCollector(column, statsCollectorConfig)
: new NoDictColumnStatisticsCollector(column, statsCollectorConfig);
for (Object value : outputValues) {
statsCollector.collect(value);
@@ -1018,6 +1013,40 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
}
+ /**
+ * Helper method to convert the output of a transform function to the appropriate type for an SV or MV
+ * {@link FieldSpec.DataType#BIG_DECIMAL} field
+ *
+ * @param outputValue the output of the transform function
+ * @param isSingleValue true if the field (column) is single-valued
+ * @param outputValueType the output value type for the transform function; can be null (in which case,
+ * the {@code outputValue} should be the field's default null value)
+ * @param defaultNullValue the default null value for the field
+ * @return the converted output value (either a BigDecimal or a BigDecimal[])
+ */
+ private Object getBigDecimalOutputValue(Object outputValue, boolean isSingleValue, PinotDataType outputValueType,
+ BigDecimal defaultNullValue) {
+ if (isSingleValue) {
+ // Skip type conversion if output value is already the required type. The outputValueType is guaranteed to be
+ // non-null if outputValue is not the default null value
+ if (outputValue instanceof BigDecimal) {
+ return outputValue;
+ } else {
+ return outputValueType.toBigDecimal(outputValue);
+ }
+ } else {
+ if (outputValue instanceof BigDecimal) {
+ return new BigDecimal[]{(BigDecimal) outputValue};
+ } else {
+ BigDecimal[] values = outputValueType.toBigDecimalArray(outputValue);
+ if (values.length == 0) {
+ values = new BigDecimal[]{defaultNullValue};
+ }
+ return values;
+ }
+ }
+ }
+
/**
* Helper method to convert the output of a transform function to the appropriate type for an SV or MV
* {@link FieldSpec.DataType#BYTES} field
@@ -1149,6 +1178,9 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
case DOUBLE:
forwardIndexCreator.putDoubleMV((double[]) outputValue);
break;
+ case BIG_DECIMAL:
+ forwardIndexCreator.putBigDecimalMV((BigDecimal[]) outputValue);
+ break;
case STRING:
forwardIndexCreator.putStringMV((String[]) outputValue);
break;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
index 49e1c5e8fa0..9caacad6745 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkForwardIndexReaderV4.java
@@ -179,6 +179,17 @@ public class VarByteChunkForwardIndexReaderV4
return ArraySerDeUtils.deserializeDoubleArrayWithLength(context.getValue(docId));
}
+ @Override
+ public int getBigDecimalMV(int docId, BigDecimal[] valueBuffer,
+ VarByteChunkForwardIndexReaderV4.ReaderContext context) {
+ return ArraySerDeUtils.deserializeBigDecimalArray(context.getValue(docId), valueBuffer);
+ }
+
+ @Override
+ public BigDecimal[] getBigDecimalMV(int docId, VarByteChunkForwardIndexReaderV4.ReaderContext context) {
+ return ArraySerDeUtils.deserializeBigDecimalArray(context.getValue(docId));
+ }
+
@Override
public int getStringMV(int docId, String[] valueBuffer, VarByteChunkForwardIndexReaderV4.ReaderContext context) {
return ArraySerDeUtils.deserializeStringArray(context.getValue(docId), valueBuffer);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkMVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkMVForwardIndexReader.java
index bd7dc5dbb74..1fb21b8097a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkMVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkMVForwardIndexReader.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.segment.local.segment.index.readers.forward;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.List;
import javax.annotation.Nullable;
@@ -54,6 +55,16 @@ public final class VarByteChunkMVForwardIndexReader extends BaseChunkForwardInde
}
}
+ @Override
+ public int getBigDecimalMV(int docId, BigDecimal[] valueBuffer, ChunkReaderContext context) {
+ return ArraySerDeUtils.deserializeBigDecimalArray(getBytes(docId, context), valueBuffer);
+ }
+
+ @Override
+ public BigDecimal[] getBigDecimalMV(int docId, ChunkReaderContext context) {
+ return ArraySerDeUtils.deserializeBigDecimalArray(getBytes(docId, context));
+ }
+
@Override
public int getStringMV(int docId, String[] valueBuffer, ChunkReaderContext context) {
return ArraySerDeUtils.deserializeStringArray(getBytes(docId, context), valueBuffer);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/DefaultValueColumnReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/DefaultValueColumnReader.java
index b67a7154419..81ccd4e837d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/DefaultValueColumnReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/DefaultValueColumnReader.java
@@ -45,6 +45,7 @@ public class DefaultValueColumnReader implements ColumnReader {
private long[] _defaultLongMV;
private float[] _defaultFloatMV;
private double[] _defaultDoubleMV;
+ private BigDecimal[] _defaultBigDecimalMV;
private String[] _defaultStringMV;
private byte[][] _defaultBytesMV;
@@ -97,6 +98,12 @@ public class DefaultValueColumnReader implements ColumnReader {
_defaultDoubleMV[i] = ((Number) defaultArray[i]).doubleValue();
}
break;
+ case BIG_DECIMAL:
+ _defaultBigDecimalMV = new BigDecimal[defaultArray.length];
+ for (int i = 0; i < defaultArray.length; i++) {
+ _defaultBigDecimalMV[i] = (BigDecimal) defaultArray[i];
+ }
+ break;
case STRING:
_defaultStringMV = new String[defaultArray.length];
for (int i = 0; i < defaultArray.length; i++) {
@@ -290,6 +297,15 @@ public class DefaultValueColumnReader implements ColumnReader {
return MultiValueResult.of(_defaultDoubleMV, null);
}
+ @Override
+ public BigDecimal[] nextBigDecimalMV() {
+ if (!hasNext()) {
+ throw new IllegalStateException("No more values available");
+ }
+ _currentIndex++;
+ return _defaultBigDecimalMV;
+ }
+
@Override
public String[] nextStringMV() {
if (!hasNext()) {
@@ -400,6 +416,12 @@ public class DefaultValueColumnReader implements ColumnReader {
return MultiValueResult.of(_defaultDoubleMV, null);
}
+ @Override
+ public BigDecimal[] getBigDecimalMV(int docId) {
+ validateDocId(docId);
+ return _defaultBigDecimalMV;
+ }
+
@Override
public String[] getStringMV(int docId) {
validateDocId(docId);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReader.java
index 7f62416cdd0..c0f4e165185 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReader.java
@@ -123,6 +123,11 @@ public class PinotSegmentColumnReader implements Closeable {
_dictionary.readDoubleValues(_dictIdBuffer, numValues, values);
return values;
}
+ case BIG_DECIMAL: {
+ BigDecimal[] values = new BigDecimal[numValues];
+ _dictionary.readBigDecimalValues(_dictIdBuffer, numValues, values);
+ return values;
+ }
case STRING: {
String[] values = new String[numValues];
_dictionary.readStringValues(_dictIdBuffer, numValues, values);
@@ -170,6 +175,8 @@ public class PinotSegmentColumnReader implements Closeable {
return ArrayUtils.toObject(_forwardIndexReader.getFloatMV(docId, _forwardIndexReaderContext));
case DOUBLE:
return ArrayUtils.toObject(_forwardIndexReader.getDoubleMV(docId, _forwardIndexReaderContext));
+ case BIG_DECIMAL:
+ return _forwardIndexReader.getBigDecimalMV(docId, _forwardIndexReaderContext);
case STRING:
return _forwardIndexReader.getStringMV(docId, _forwardIndexReaderContext);
case BYTES:
@@ -289,6 +296,17 @@ public class PinotSegmentColumnReader implements Closeable {
}
}
+ public BigDecimal[] getBigDecimalMV(int docId) {
+ if (_dictionary != null) {
+ int numValues = _forwardIndexReader.getDictIdMV(docId, _dictIdBuffer, _forwardIndexReaderContext);
+ BigDecimal[] values = new BigDecimal[numValues];
+ _dictionary.readBigDecimalValues(_dictIdBuffer, numValues, values);
+ return values;
+ } else {
+ return _forwardIndexReader.getBigDecimalMV(docId, _forwardIndexReaderContext);
+ }
+ }
+
public String[] getStringMV(int docId) {
if (_dictionary != null) {
int numValues = _forwardIndexReader.getDictIdMV(docId, _dictIdBuffer, _forwardIndexReaderContext);
@@ -354,6 +372,14 @@ public class PinotSegmentColumnReader implements Closeable {
values[i] = value;
}
break;
+ case BIG_DECIMAL:
+ for (int i = 0; i < numDocs; i++) {
+ int numValues = _forwardIndexReader.getDictIdMV(i, _dictIdBuffer, _forwardIndexReaderContext);
+ BigDecimal[] value = new BigDecimal[numValues];
+ _dictionary.readBigDecimalValues(_dictIdBuffer, numValues, value);
+ values[i] = value;
+ }
+ break;
case STRING:
for (int i = 0; i < numDocs; i++) {
int numValues = _forwardIndexReader.getDictIdMV(i, _dictIdBuffer, _forwardIndexReaderContext);
@@ -442,6 +468,11 @@ public class PinotSegmentColumnReader implements Closeable {
values[i] = _forwardIndexReader.getDoubleMV(i, _forwardIndexReaderContext);
}
break;
+ case BIG_DECIMAL:
+ for (int i = 0; i < numDocs; i++) {
+ values[i] = _forwardIndexReader.getBigDecimalMV(i, _forwardIndexReaderContext);
+ }
+ break;
case STRING:
for (int i = 0; i < numDocs; i++) {
values[i] = _forwardIndexReader.getStringMV(i, _forwardIndexReaderContext);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImpl.java
index 8226408266c..0f3b30e8cac 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImpl.java
@@ -262,6 +262,14 @@ public class PinotSegmentColumnReaderImpl implements ColumnReader {
return MultiValueResult.of(_segmentColumnReader.getDoubleMV(_nextDocId++), null);
}
+ @Override
+ public BigDecimal[] nextBigDecimalMV() {
+ if (!hasNext()) {
+ throw new IllegalStateException("No more values available");
+ }
+ return _segmentColumnReader.getBigDecimalMV(_nextDocId++);
+ }
+
@Override
public String[] nextStringMV() {
if (!hasNext()) {
@@ -370,6 +378,11 @@ public class PinotSegmentColumnReaderImpl implements ColumnReader {
return MultiValueResult.of(_segmentColumnReader.getDoubleMV(docId), null);
}
+ @Override
+ public BigDecimal[] getBigDecimalMV(int docId) {
+ return _segmentColumnReader.getBigDecimalMV(docId);
+ }
+
@Override
public String[] getStringMV(int docId) {
return _segmentColumnReader.getStringMV(docId);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/ArraySerDeUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/ArraySerDeUtils.java
index 58238a33e06..482135a13b0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/ArraySerDeUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/ArraySerDeUtils.java
@@ -18,7 +18,9 @@
*/
package org.apache.pinot.segment.local.utils;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -279,6 +281,45 @@ public class ArraySerDeUtils {
}
}
+ public static byte[] serializeBigDecimalArray(BigDecimal[] values) {
+ // Format: [numValues][length1][length2]...[lengthN][value1][value2]...[valueN] where each value is a
+ // BigDecimalUtils.serializeWithSize payload (scale + unscaled bytes).
+ int headerSize = Integer.BYTES + Integer.BYTES * values.length;
+ int size = headerSize;
+ byte[][] serialized = new byte[values.length][];
+ for (int i = 0; i < values.length; i++) {
+ serialized[i] = BigDecimalUtils.serialize(values[i]);
+ size += serialized[i].length;
+ }
+ return writeValues(serialized, size, headerSize);
+ }
+
+ public static BigDecimal[] deserializeBigDecimalArray(byte[] bytes) {
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ int numValues = byteBuffer.getInt();
+ BigDecimal[] values = new BigDecimal[numValues];
+ readValues(bytes, byteBuffer, values, numValues);
+ return values;
+ }
+
+ public static int deserializeBigDecimalArray(byte[] bytes, BigDecimal[] values) {
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
+ int numValues = byteBuffer.getInt();
+ readValues(bytes, byteBuffer, values, numValues);
+ return numValues;
+ }
+
+ private static void readValues(byte[] bytes, ByteBuffer byteBuffer, BigDecimal[] values, int numValues) {
+ int offset = (numValues + 1) * Integer.BYTES;
+ for (int i = 0; i < numValues; i++) {
+ int length = byteBuffer.getInt();
+ byte[] valueBytes = new byte[length];
+ System.arraycopy(bytes, offset, valueBytes, 0, length);
+ values[i] = BigDecimalUtils.deserialize(valueBytes);
+ offset += length;
+ }
+ }
+
public static byte[] serializeStringArray(String[] values) {
// Format: [numValues][length1][length2]...[lengthN][value1][value2]...[valueN]
int headerSize = Integer.BYTES + Integer.BYTES * values.length;
@@ -306,6 +347,15 @@ public class ArraySerDeUtils {
return numValues;
}
+ private static void readValues(byte[] bytes, ByteBuffer byteBuffer, String[] values, int numValues) {
+ int offset = (numValues + 1) * Integer.BYTES;
+ for (int i = 0; i < numValues; i++) {
+ int length = byteBuffer.getInt();
+ values[i] = new String(bytes, offset, length, UTF_8);
+ offset += length;
+ }
+ }
+
public static byte[] serializeBytesArray(byte[][] values) {
// Format: [numValues][length1][length2]...[lengthN][value1][value2]...[valueN]
int headerSize = Integer.BYTES + Integer.BYTES * values.length;
@@ -331,6 +381,15 @@ public class ArraySerDeUtils {
return numValues;
}
+ private static void readValues(ByteBuffer byteBuffer, byte[][] values, int numValues) {
+ byteBuffer.position((numValues + 1) * Integer.BYTES);
+ for (int i = 0; i < numValues; i++) {
+ int length = byteBuffer.getInt((i + 1) * Integer.BYTES);
+ values[i] = new byte[length];
+ byteBuffer.get(values[i]);
+ }
+ }
+
private static byte[] writeValues(byte[][] values, int size, int headerSize) {
byte[] bytes = new byte[size];
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
@@ -342,22 +401,4 @@ public class ArraySerDeUtils {
}
return bytes;
}
-
- private static void readValues(byte[] bytes, ByteBuffer byteBuffer, String[] values, int numValues) {
- int offset = (numValues + 1) * Integer.BYTES;
- for (int i = 0; i < numValues; i++) {
- int length = byteBuffer.getInt();
- values[i] = new String(bytes, offset, length, UTF_8);
- offset += length;
- }
- }
-
- private static void readValues(ByteBuffer byteBuffer, byte[][] values, int numValues) {
- byteBuffer.position((numValues + 1) * Integer.BYTES);
- for (int i = 0; i < numValues; i++) {
- int length = byteBuffer.getInt((i + 1) * Integer.BYTES);
- values[i] = new byte[length];
- byteBuffer.get(values[i]);
- }
- }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java
index b1df94fe704..b381d73d131 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java
@@ -174,8 +174,6 @@ public class SchemaUtils {
private static void validateMultiValueCompatibility(FieldSpec fieldSpec) {
Preconditions.checkState(!fieldSpec.getDataType().equals(FieldSpec.DataType.JSON),
"JSON columns cannot be of multi-value type");
- Preconditions.checkState(!fieldSpec.getDataType().equals(FieldSpec.DataType.BIG_DECIMAL),
- "BIG_DECIMAL columns cannot be of multi-value type");
}
/**
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarRowMajorEquivalenceTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarRowMajorEquivalenceTest.java
index ac2cac33f7b..7cb684ccaf6 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarRowMajorEquivalenceTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarRowMajorEquivalenceTest.java
@@ -76,6 +76,7 @@ public class ColumnarRowMajorEquivalenceTest extends ColumnarSegmentBuildingTest
Assert.assertTrue(columnNames.contains(BIG_DECIMAL_COL), "BIG_DECIMAL column missing");
Assert.assertTrue(columnNames.contains(BYTES_COL), "BYTES column missing");
Assert.assertTrue(columnNames.contains(MV_INT_COL), "Multi-value INT column missing");
+ Assert.assertTrue(columnNames.contains(MV_BIG_DECIMAL_COL), "Multi-value BIG_DECIMAL column missing");
Assert.assertTrue(columnNames.contains(MV_STRING_COL), "Multi-value STRING column missing");
Assert.assertTrue(columnNames.contains(TIME_COL), "TIME column missing");
} finally {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarSegmentBuildingTestBase.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarSegmentBuildingTestBase.java
index 81f51f98e43..cef8ebea9f5 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarSegmentBuildingTestBase.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnarSegmentBuildingTestBase.java
@@ -80,6 +80,7 @@ public abstract class ColumnarSegmentBuildingTestBase {
protected static final String MV_LONG_COL = "mvLongCol";
protected static final String MV_FLOAT_COL = "mvFloatCol";
protected static final String MV_DOUBLE_COL = "mvDoubleCol";
+ protected static final String MV_BIG_DECIMAL_COL = "mvBigDecimalCol";
protected static final String MV_STRING_COL = "mvStringCol";
protected static final String MV_BYTES_COL = "mvBytesCol";
@@ -116,6 +117,7 @@ public abstract class ColumnarSegmentBuildingTestBase {
.addMultiValueDimension(MV_LONG_COL, FieldSpec.DataType.LONG)
.addMultiValueDimension(MV_FLOAT_COL, FieldSpec.DataType.FLOAT)
.addMultiValueDimension(MV_DOUBLE_COL, FieldSpec.DataType.DOUBLE)
+ .addMultiValueDimension(MV_BIG_DECIMAL_COL, FieldSpec.DataType.BIG_DECIMAL)
.addMultiValueDimension(MV_STRING_COL, FieldSpec.DataType.STRING)
.addMultiValueDimension(MV_BYTES_COL, FieldSpec.DataType.BYTES)
.addDateTime(TIME_COL, FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
@@ -136,6 +138,7 @@ public abstract class ColumnarSegmentBuildingTestBase {
.addMultiValueDimension(MV_LONG_COL, FieldSpec.DataType.LONG)
.addMultiValueDimension(MV_FLOAT_COL, FieldSpec.DataType.FLOAT)
.addMultiValueDimension(MV_DOUBLE_COL, FieldSpec.DataType.DOUBLE)
+ .addMultiValueDimension(MV_BIG_DECIMAL_COL, FieldSpec.DataType.BIG_DECIMAL)
.addMultiValueDimension(MV_STRING_COL, FieldSpec.DataType.STRING)
.addMultiValueDimension(MV_BYTES_COL, FieldSpec.DataType.BYTES)
.addDateTime(TIME_COL, FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
@@ -200,6 +203,7 @@ public abstract class ColumnarSegmentBuildingTestBase {
allColumns.add(MV_LONG_COL);
allColumns.add(MV_FLOAT_COL);
allColumns.add(MV_DOUBLE_COL);
+ allColumns.add(MV_BIG_DECIMAL_COL);
allColumns.add(MV_STRING_COL);
allColumns.add(MV_BYTES_COL);
@@ -238,6 +242,10 @@ public abstract class ColumnarSegmentBuildingTestBase {
spec.setSingleValueField(false);
spec.setNullable(true);
})
+ .addDimensionField(MV_BIG_DECIMAL_COL, FieldSpec.DataType.BIG_DECIMAL, spec -> {
+ spec.setSingleValueField(false);
+ spec.setNullable(true);
+ })
.addDimensionField(MV_STRING_COL, FieldSpec.DataType.STRING, spec -> {
spec.setSingleValueField(false);
spec.setNullable(true);
@@ -511,6 +519,8 @@ public abstract class ColumnarSegmentBuildingTestBase {
: new Object[]{(float) i * 1.1f, (float) (i + 1) * 1.1f, (float) (i + 2) * 1.1f});
row.putValue(MV_DOUBLE_COL, random.nextDouble() < nullProbability ? null
: new Object[]{(double) i * 2.2, (double) (i + 1) * 2.2, (double) (i + 2) * 2.2});
+ row.putValue(MV_BIG_DECIMAL_COL, random.nextDouble() < nullProbability ? null
+ : new Object[]{new BigDecimal(i + ".1"), new BigDecimal((i + 1) + ".2"), new BigDecimal((i + 2) + ".3")});
row.putValue(MV_STRING_COL, random.nextDouble() < nullProbability ? null
: new Object[]{"mv1_" + i, "mv2_" + i, "mv3_" + (i % 3)});
row.putValue(MV_BYTES_COL, random.nextDouble() < nullProbability ? null
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
index d5d6a9348ed..0752de3beee 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
@@ -30,6 +30,7 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.BooleanUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
@@ -139,7 +140,8 @@ public class AbstractColumnStatisticsCollectorTest {
@DataProvider(name = "multiValueSupportedTypes")
public Object[][] multiValueSupportedTypes() {
return new Object[][]{
- {DataType.INT}, {DataType.LONG}, {DataType.FLOAT}, {DataType.DOUBLE}, {DataType.STRING}, {DataType.BYTES}
+ {DataType.INT}, {DataType.LONG}, {DataType.FLOAT}, {DataType.DOUBLE}, {DataType.BIG_DECIMAL},
+ {DataType.STRING}, {DataType.BYTES}
};
}
@@ -286,6 +288,10 @@ public class AbstractColumnStatisticsCollectorTest {
collector.collect(new double[]{1.5, 2.5});
collector.collect(new double[]{3.5, 4.5, 5.5});
break;
+ case BIG_DECIMAL:
+ collector.collect(new BigDecimal[]{new BigDecimal("1.5"), new BigDecimal("2.5")});
+ collector.collect(new BigDecimal[]{new BigDecimal("3.5"), new BigDecimal("4.5"), new BigDecimal("5.5")});
+ break;
case STRING:
collector.collect(new String[]{"a", "b"});
collector.collect(new String[]{"c", "d", "e"});
@@ -437,12 +443,21 @@ public class AbstractColumnStatisticsCollectorTest {
assertTrue(collector.getLengthOfLongestElement() >= collector.getLengthOfShortestElement());
}
- @Test(expectedExceptions = UnsupportedOperationException.class)
- public void testBigDecimalMultiValueNotSupported() {
- AbstractColumnStatisticsCollector collector = createCollector(DataType.BIG_DECIMAL, true, false);
+ @Test
+ public void testBigDecimalMultiValueLengthTracking() {
+ AbstractColumnStatisticsCollector collector = createCollector(DataType.BIG_DECIMAL, false, false);
+
+ int shortLength = BigDecimalUtils.byteSize(new BigDecimal("1.0"));
+ int midLength = BigDecimalUtils.byteSize(new BigDecimal("99.99"));
+ int longLength = BigDecimalUtils.byteSize(new BigDecimal("123456.789"));
+ collector.collect(new BigDecimal[]{new BigDecimal("1.0"), new BigDecimal("99.99")});
+ collector.collect(new BigDecimal[]{new BigDecimal("123456.789"), new BigDecimal("99.99")});
- // BigDecimal does not support multi-value
- collector.collect(new Object[]{new BigDecimal("1.0"), new BigDecimal("2.0")});
+ collector.seal();
+
+ assertEquals(collector.getLengthOfShortestElement(), Math.min(shortLength, midLength));
+ assertEquals(collector.getLengthOfLongestElement(), longLength);
+ assertEquals(collector.getMaxRowLengthInBytes(), longLength + midLength);
}
@Test
@@ -773,6 +788,11 @@ public class AbstractColumnStatisticsCollectorTest {
return new Object[][]{
{1.5, 2.5}, {3.5, 4.5, 5.5}
};
+ case BIG_DECIMAL:
+ return new Object[][]{
+ {new BigDecimal("1.5"), new BigDecimal("2.5")},
+ {new BigDecimal("3.5"), new BigDecimal("4.5"), new BigDecimal("5.5")}
+ };
case STRING:
return new Object[][]{
{"a", "b"}, {"c", "d", "e"}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
index 04a08534fd2..0cd1d95a333 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
@@ -47,8 +47,8 @@ public class EmptyColumnStatisticsTest {
List<Object[]> params = new ArrayList<>();
for (DataType type : ALL_TYPES) {
params.add(new Object[]{type, true});
- // BIG_DECIMAL and MAP do not support MV
- if (type != DataType.BIG_DECIMAL && type != DataType.MAP) {
+ // MAP does not support MV
+ if (type != DataType.MAP) {
params.add(new Object[]{type, false});
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollectorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollectorTest.java
index f0f49daa657..df2b335d7c8 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollectorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollectorTest.java
@@ -445,15 +445,13 @@ public class NoDictColumnStatisticsCollectorTest {
}
private void runRandomizedTest(Random random, int iteration) {
- // Randomly select data type (excluding BIG_DECIMAL for MV as it's unsupported)
DataType[] supportedTypes = {
DataType.INT, DataType.LONG, DataType.FLOAT, DataType.DOUBLE, DataType.BIG_DECIMAL, DataType.STRING,
DataType.BYTES
};
DataType dataType = supportedTypes[random.nextInt(supportedTypes.length)];
- // Randomly choose single vs multi-value (skip MV for BIG_DECIMAL)
- boolean isSingleValue = dataType == DataType.BIG_DECIMAL || random.nextBoolean();
+ boolean isSingleValue = random.nextBoolean();
// Randomly choose cardinality level
CardinalityLevel cardinalityLevel = CardinalityLevel.values()[random.nextInt(CardinalityLevel.values().length)];
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
index f45fb7f46c9..6530f312873 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/MultiValueVarByteRawIndexCreatorTest.java
@@ -20,6 +20,8 @@ package org.apache.pinot.segment.local.segment.index.creator;
import java.io.File;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
@@ -95,6 +97,60 @@ public class MultiValueVarByteRawIndexCreatorTest implements PinotBuffersAfterMe
ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes(), ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
+ @Test(dataProvider = "params")
+ public void testMVBigDecimal(ChunkCompressionType compressionType, boolean useFullSize, int writerVersion,
+ int maxLength, int maxNumEntries)
+ throws IOException {
+ String column = "testCol-" + UUID.randomUUID();
+ int numDocs = 1000;
+ File file = new File(OUTPUT_DIR, column + Indexes.RAW_MV_FORWARD_INDEX_FILE_EXTENSION);
+ List<BigDecimal[]> inputs = new ArrayList<>();
+ Random random = new Random();
+ int maxTotalLength = 0;
+ int maxElements = 0;
+ for (int i = 0; i < numDocs; i++) {
+ int numEntries = useFullSize ? maxNumEntries : random.nextInt(maxNumEntries + 1);
+ maxElements = Math.max(numEntries, maxElements);
+ BigDecimal[] values = new BigDecimal[numEntries];
+ int serializedLength = 0;
+ for (int j = 0; j < numEntries; j++) {
+ // BigDecimal serialized size = 2 bytes (scale) + N bytes (unscaled). Bound the unscaled byte length by
+ // maxLength so we exercise the maxRowLengthInBytes accounting.
+ int maxUnscaledBytes = Math.max(1, maxLength - 2);
+ int unscaledByteLength = useFullSize ? maxUnscaledBytes : random.nextInt(maxUnscaledBytes) + 1;
+ byte[] unscaledBytes = new byte[unscaledByteLength];
+ random.nextBytes(unscaledBytes);
+ BigInteger unscaled = new BigInteger(unscaledBytes);
+ int scale = random.nextInt(10);
+ values[j] = new BigDecimal(unscaled, scale);
+ serializedLength += 2 + unscaledByteLength;
+ }
+ maxTotalLength = Math.max(serializedLength, maxTotalLength);
+ inputs.add(values);
+ }
+ try (MultiValueVarByteRawIndexCreator creator = new MultiValueVarByteRawIndexCreator(OUTPUT_DIR, compressionType,
+ column, numDocs, DataType.BIG_DECIMAL, writerVersion, maxTotalLength, maxElements, 1024 * 1024, 1000)) {
+ for (BigDecimal[] input : inputs) {
+ creator.putBigDecimalMV(input);
+ }
+ }
+
+ try (PinotDataBuffer buffer = PinotDataBuffer.mapFile(file, true, 0, file.length(), ByteOrder.BIG_ENDIAN, "");
+ ForwardIndexReader reader = ForwardIndexReaderFactory.getInstance()
+ .createRawIndexReader(buffer, DataType.BIG_DECIMAL, false);
+ ForwardIndexReaderContext context = reader.createContext()) {
+ BigDecimal[] values = new BigDecimal[maxElements];
+ for (int i = 0; i < numDocs; i++) {
+ BigDecimal[] input = inputs.get(i);
+ assertEquals(reader.getNumValuesMV(i, context), input.length);
+ int length = reader.getBigDecimalMV(i, values, context);
+ assertEquals(Arrays.copyOf(values, length), input);
+ // Exercise the alternate getter that allocates the result array too.
+ assertEquals(reader.getBigDecimalMV(i, context), input);
+ }
+ }
+ }
+
@Test(dataProvider = "params")
public void testMVString(ChunkCompressionType compressionType, boolean useFullSize, int writerVersion, int maxLength,
int maxNumEntries)
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
index 31ef7939794..6023d53c278 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/InMemoryColumnReader.java
@@ -167,6 +167,11 @@ public class InMemoryColumnReader implements ColumnReader {
return MultiValueResult.of((double[]) next(), null);
}
+ @Override
+ public BigDecimal[] nextBigDecimalMV() {
+ return (BigDecimal[]) next();
+ }
+
@Override
public String[] nextStringMV() {
return (String[]) next();
@@ -257,6 +262,11 @@ public class InMemoryColumnReader implements ColumnReader {
return MultiValueResult.of((double[]) _values[docId], null);
}
+ @Override
+ public BigDecimal[] getBigDecimalMV(int docId) {
+ return (BigDecimal[]) _values[docId];
+ }
+
@Override
public String[] getStringMV(int docId) {
return (String[]) _values[docId];
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImplTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImplTest.java
index 189848f04d0..f7442a42df0 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImplTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentColumnReaderImplTest.java
@@ -427,6 +427,8 @@ public class PinotSegmentColumnReaderImplTest extends ColumnarSegmentBuildingTes
Assert.assertFalse(result.hasNulls(), "Multi-value primitive types should not have nulls");
return result.getValues();
}, null},
+ {MV_BIG_DECIMAL_COL, (MultiValueGetter) PinotSegmentColumnReaderImpl::getBigDecimalMV,
+ (MultiValueSequentialGetter) PinotSegmentColumnReaderImpl::nextBigDecimalMV, null},
{MV_STRING_COL, (MultiValueGetter) PinotSegmentColumnReaderImpl::getStringMV,
(MultiValueSequentialGetter) PinotSegmentColumnReaderImpl::nextStringMV, null},
{MV_BYTES_COL, (MultiValueGetter) PinotSegmentColumnReaderImpl::getBytesMV,
@@ -492,6 +494,9 @@ public class PinotSegmentColumnReaderImplTest extends ColumnarSegmentBuildingTes
case DOUBLE:
defaultValue = new double[]{((Number) defaultNullValue).doubleValue()};
break;
+ case BIG_DECIMAL:
+ defaultValue = new BigDecimal[]{(BigDecimal) defaultNullValue};
+ break;
case STRING:
defaultValue = new String[]{(String) defaultNullValue};
break;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
index 3c17c74e651..97bc18e985f 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
@@ -21,6 +21,7 @@ package org.apache.pinot.segment.spi.index;
import java.io.Closeable;
import java.io.IOException;
+import java.math.BigDecimal;
import javax.annotation.Nullable;
@@ -137,6 +138,11 @@ public interface IndexCreator extends Closeable {
add(boxedValues, dictIds);
}
+ default void addBigDecimalMV(BigDecimal[] values, @Nullable int[] dictIds)
+ throws IOException {
+ add(values, dictIds);
+ }
+
default void addStringMV(String[] values, @Nullable int[] dictIds)
throws IOException {
add(values, dictIds);
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
index 32243e6ecbe..8de96458e0b 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
@@ -122,6 +122,17 @@ public interface ForwardIndexCreator extends IndexCreator {
}
putDoubleMV(doubles);
break;
+ case BIG_DECIMAL:
+ if (cellValues instanceof BigDecimal[]) {
+ putBigDecimalMV((BigDecimal[]) cellValues);
+ } else {
+ BigDecimal[] bigDecimals = new BigDecimal[length];
+ for (int i = 0; i < length; i++) {
+ bigDecimals[i] = (BigDecimal) cellValues[i];
+ }
+ putBigDecimalMV(bigDecimals);
+ }
+ break;
case STRING:
if (cellValues instanceof String[]) {
putStringMV((String[]) cellValues);
@@ -256,6 +267,16 @@ public interface ForwardIndexCreator extends IndexCreator {
}
}
+ @Override
+ default void addBigDecimalMV(BigDecimal[] values, @Nullable int[] dictIds)
+ throws IOException {
+ if (dictIds != null) {
+ putDictIdMV(dictIds);
+ } else {
+ putBigDecimalMV(values);
+ }
+ }
+
@Override
default void addStringMV(String[] values, @Nullable int[] dictIds)
throws IOException {
@@ -423,6 +444,15 @@ public interface ForwardIndexCreator extends IndexCreator {
throw new UnsupportedOperationException();
}
+ /**
+ * Writes the next BIG_DECIMAL type multi-value into the forward index.
+ *
+ * @param values Values to write
+ */
+ default void putBigDecimalMV(BigDecimal[] values) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Writes the next STRING type multi-value into the forward index.
*
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/MutableForwardIndex.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/MutableForwardIndex.java
index 627d29e8faa..676f6410ef4 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/MutableForwardIndex.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/MutableForwardIndex.java
@@ -129,6 +129,17 @@ public interface MutableForwardIndex extends ForwardIndexReader<ForwardIndexRead
}
setDoubleMV(docId, doubles);
break;
+ case BIG_DECIMAL:
+ if (value instanceof BigDecimal[]) {
+ setBigDecimalMV(docId, (BigDecimal[]) value);
+ } else {
+ BigDecimal[] bigDecimals = new BigDecimal[length];
+ for (int i = 0; i < length; i++) {
+ bigDecimals[i] = (BigDecimal) value[i];
+ }
+ setBigDecimalMV(docId, bigDecimals);
+ }
+ break;
case STRING:
if (value instanceof String[]) {
setStringMV(docId, (String[]) value);
@@ -707,6 +718,16 @@ public interface MutableForwardIndex extends ForwardIndexReader<ForwardIndexRead
throw new UnsupportedOperationException();
}
+ /**
+ * Writes the BIG_DECIMAL type multi-value into the given document id.
+ *
+ * @param docId Document id
+ * @param values Values to write
+ */
+ default void setBigDecimalMV(int docId, BigDecimal[] values) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Writes the STRING type multi-value into the given document id.
*
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java
index 0b44dc8fddc..d05078494cc 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java
@@ -583,6 +583,16 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
}
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValueBuffer = new BigDecimal[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getBigDecimalMV(docIds[i], bigDecimalValueBuffer, context);
+ values[i] = new int[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i][j] = bigDecimalValueBuffer[j].intValue();
+ }
+ }
+ break;
case STRING:
String[] stringValueBuffer = new String[maxNumValuesPerMVEntry];
for (int i = 0; i < length; i++) {
@@ -643,6 +653,16 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
}
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValueBuffer = new BigDecimal[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getBigDecimalMV(docIds[i], bigDecimalValueBuffer, context);
+ values[i] = new long[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i][j] = bigDecimalValueBuffer[j].longValue();
+ }
+ }
+ break;
case STRING:
String[] stringValueBuffer = new String[maxNumValuesPerMVEntry];
for (int i = 0; i < length; i++) {
@@ -703,6 +723,16 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
}
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValueBuffer = new BigDecimal[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getBigDecimalMV(docIds[i], bigDecimalValueBuffer, context);
+ values[i] = new float[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i][j] = bigDecimalValueBuffer[j].floatValue();
+ }
+ }
+ break;
case STRING:
String[] stringValueBuffer = new String[maxNumValuesPerMVEntry];
for (int i = 0; i < length; i++) {
@@ -763,6 +793,16 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
values[i] = getDoubleMV(docIds[i], context);
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValueBuffer = new BigDecimal[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getBigDecimalMV(docIds[i], bigDecimalValueBuffer, context);
+ values[i] = new double[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i][j] = bigDecimalValueBuffer[j].doubleValue();
+ }
+ }
+ break;
case STRING:
String[] stringValueBuffer = new String[maxNumValuesPerMVEntry];
for (int i = 0; i < length; i++) {
@@ -828,6 +868,16 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
}
}
break;
+ case BIG_DECIMAL:
+ BigDecimal[] bigDecimalValueBuffer = new BigDecimal[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getBigDecimalMV(docIds[i], bigDecimalValueBuffer, context);
+ values[i] = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i][j] = bigDecimalValueBuffer[j].toPlainString();
+ }
+ }
+ break;
case STRING:
for (int i = 0; i < length; i++) {
values[i] = getStringMV(docIds[i], context);
@@ -838,6 +888,86 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
}
}
+ /**
+ * Fills the values
+ * @param docIds Array containing the document ids to read
+ * @param length Number of values to read
+ * @param maxNumValuesPerMVEntry Maximum number of values per MV entry
+ * @param values Values to fill
+ * @param context Reader context
+ */
+ default void readValuesMV(int[] docIds, int length, int maxNumValuesPerMVEntry, BigDecimal[][] values, T context) {
+ switch (getStoredType()) {
+ case INT: {
+ int[] valueBuffer = new int[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getIntMV(docIds[i], valueBuffer, context);
+ BigDecimal[] row = new BigDecimal[numValues];
+ for (int j = 0; j < numValues; j++) {
+ row[j] = BigDecimal.valueOf(valueBuffer[j]);
+ }
+ values[i] = row;
+ }
+ break;
+ }
+ case LONG: {
+ long[] valueBuffer = new long[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getLongMV(docIds[i], valueBuffer, context);
+ BigDecimal[] row = new BigDecimal[numValues];
+ for (int j = 0; j < numValues; j++) {
+ row[j] = BigDecimal.valueOf(valueBuffer[j]);
+ }
+ values[i] = row;
+ }
+ break;
+ }
+ case FLOAT: {
+ float[] valueBuffer = new float[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getFloatMV(docIds[i], valueBuffer, context);
+ BigDecimal[] row = new BigDecimal[numValues];
+ for (int j = 0; j < numValues; j++) {
+ row[j] = BigDecimal.valueOf(valueBuffer[j]);
+ }
+ values[i] = row;
+ }
+ break;
+ }
+ case DOUBLE: {
+ double[] valueBuffer = new double[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getDoubleMV(docIds[i], valueBuffer, context);
+ BigDecimal[] row = new BigDecimal[numValues];
+ for (int j = 0; j < numValues; j++) {
+ row[j] = BigDecimal.valueOf(valueBuffer[j]);
+ }
+ values[i] = row;
+ }
+ break;
+ }
+ case BIG_DECIMAL:
+ for (int i = 0; i < length; i++) {
+ values[i] = getBigDecimalMV(docIds[i], context);
+ }
+ break;
+ case STRING: {
+ String[] valueBuffer = new String[maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = getStringMV(docIds[i], valueBuffer, context);
+ BigDecimal[] row = new BigDecimal[numValues];
+ for (int j = 0; j < numValues; j++) {
+ row[j] = new BigDecimal(valueBuffer[j]);
+ }
+ values[i] = row;
+ }
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("readValuesMV not supported for type " + getStoredType());
+ }
+ }
+
/**
* Fills the values
* @param docIds Array containing the document ids to read
@@ -952,6 +1082,31 @@ public interface ForwardIndexReader<T extends ForwardIndexReaderContext> extends
throw new UnsupportedOperationException();
}
+ /**
+ * Reads the BIG_DECIMAL type multi-value at the given document id into the passed in value buffer (the buffer size
+ * must be enough to hold all the values for the multi-value entry) and returns the number of values within the
+ * multi-value entry.
+ *
+ * @param docId Document id
+ * @param valueBuffer Value buffer
+ * @param context Reader context
+ * @return Number of values within the multi-value entry
+ */
+ default int getBigDecimalMV(int docId, BigDecimal[] valueBuffer, T context) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Reads the BIG_DECIMAL type multi-value at the given document id.
+ *
+ * @param docId Document id
+ * @param context Reader context
+ * @return BIG_DECIMAL values at the given document id
+ */
+ default BigDecimal[] getBigDecimalMV(int docId, T context) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Reads the STRING type multi-value at the given document id into the passed in value buffer (the buffer size must
* be enough to hold all the values for the multi-value entry) and returns the number of values within the multi-value
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/ColumnReader.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/ColumnReader.java
index faf2bb11ac4..d6f4deac054 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/ColumnReader.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/ColumnReader.java
@@ -264,6 +264,9 @@ public interface ColumnReader extends Closeable, Serializable {
MultiValueResult<double[]> nextDoubleMV()
throws IOException;
+ BigDecimal[] nextBigDecimalMV()
+ throws IOException;
+
String[] nextStringMV()
throws IOException;
@@ -378,6 +381,9 @@ public interface ColumnReader extends Closeable, Serializable {
MultiValueResult<double[]> getDoubleMV(int docId)
throws IOException;
+ BigDecimal[] getBigDecimalMV(int docId)
+ throws IOException;
+
String[] getStringMV(int docId)
throws IOException;
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java
index c4b8f354982..6c41b45a232 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java
@@ -492,6 +492,63 @@ public class ArrayCopyUtils {
}
}
+ public static void copy(BigDecimal[][] src, int[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new int[rowLength];
+ copy(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copy(BigDecimal[][] src, long[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new long[rowLength];
+ copy(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copy(BigDecimal[][] src, float[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new float[rowLength];
+ copy(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copy(BigDecimal[][] src, double[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new double[rowLength];
+ copy(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copyToBoolean(BigDecimal[][] src, int[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new int[rowLength];
+ copyToBoolean(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copy(BigDecimal[][] src, String[][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ dest[i] = new String[rowLength];
+ copy(src[i], dest[i], rowLength);
+ }
+ }
+
+ public static void copy(BigDecimal[][] src, byte[][][] dest, int length) {
+ for (int i = 0; i < length; i++) {
+ int rowLength = src[i].length;
+ byte[][] bytesValues = new byte[rowLength][];
+ copy(src[i], bytesValues, rowLength);
+ dest[i] = bytesValues;
+ }
+ }
+
public static void copyFromBoolean(int[][] src, String[][] dest, int length) {
for (int i = 0; i < length; i++) {
int rowLength = src[i].length;
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index a3d8f9a6663..7a056576112 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -20,8 +20,8 @@ package org.apache.pinot.spi.utils;
import java.io.File;
import java.math.BigDecimal;
-import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
@@ -2252,9 +2252,11 @@ public class CommonConstants {
public static final long[] LONG_ARRAY = new long[0];
public static final float[] FLOAT_ARRAY = new float[0];
public static final double[] DOUBLE_ARRAY = new double[0];
+ public static final BigDecimal[] BIG_DECIMAL_ARRAY = new BigDecimal[0];
public static final String[] STRING_ARRAY = new String[0];
public static final byte[][] BYTES_ARRAY = new byte[0][];
- public static final Object MAP = Collections.emptyMap();
+ public static final ByteArray[] INTERNAL_BYTES_ARRAY = new ByteArray[0];
+ public static final Object MAP = Map.of();
}
public static class CursorConfigs {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org