You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/10/23 16:14:42 UTC
[pinot] branch master updated: fixing arrayAgg null support (#11853)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8166842606 fixing arrayAgg null support (#11853)
8166842606 is described below
commit 81668426069df5ddb5ac03b27d050adb94de4525
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Mon Oct 23 09:14:35 2023 -0700
fixing arrayAgg null support (#11853)
---
.../array/ArrayAggDistinctDoubleFunction.java | 16 ++--
.../array/ArrayAggDistinctFloatFunction.java | 16 ++--
.../array/ArrayAggDistinctIntFunction.java | 23 +++---
.../array/ArrayAggDistinctLongFunction.java | 13 ++--
.../array/ArrayAggDistinctStringFunction.java | 18 ++---
.../function/array/ArrayAggDoubleFunction.java | 14 ++--
.../function/array/ArrayAggFloatFunction.java | 14 ++--
.../function/array/ArrayAggIntFunction.java | 12 +--
.../function/array/ArrayAggLongFunction.java | 13 ++--
.../function/array/ArrayAggStringFunction.java | 16 ++--
.../function/array/BaseArrayAggFunction.java | 7 +-
.../pinot/integration/tests/custom/ArrayTest.java | 89 +++++++++++++++++++---
12 files changed, 145 insertions(+), 106 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctDoubleFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctDoubleFunction.java
index 1493ad82aa..f5dacc3f4b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctDoubleFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctDoubleFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -56,15 +55,12 @@ public class ArrayAggDistinctDoubleFunction extends BaseArrayAggDoubleFunction<D
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, double value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- DoubleOpenHashSet valueArray = new DoubleOpenHashSet();
- valueArray.add(value);
- resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- DoubleOpenHashSet valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, double value) {
+ DoubleOpenHashSet valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new DoubleOpenHashSet();
+ resultHolder.setValueForKey(groupKey, valueSet);
}
+ valueSet.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctFloatFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctFloatFunction.java
index d9ad31ca35..cf3e21eeb1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctFloatFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctFloatFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -56,15 +55,12 @@ public class ArrayAggDistinctFloatFunction extends BaseArrayAggFloatFunction<Flo
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, float value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- FloatOpenHashSet valueArray = new FloatOpenHashSet();
- valueArray.add(value);
- resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- FloatOpenHashSet valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, float value) {
+ FloatOpenHashSet valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new FloatOpenHashSet();
+ resultHolder.setValueForKey(groupKey, valueSet);
}
+ valueSet.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctIntFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctIntFunction.java
index 3aa757e311..1ccbb27b0a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctIntFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctIntFunction.java
@@ -22,9 +22,7 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
-import org.apache.pinot.core.query.aggregation.ObjectAggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.apache.pinot.spi.data.FieldSpec;
import org.roaringbitmap.RoaringBitmap;
@@ -38,19 +36,17 @@ public class ArrayAggDistinctIntFunction extends BaseArrayAggIntFunction<IntOpen
@Override
protected void aggregateArray(int length, AggregationResultHolder aggregationResultHolder,
BlockValSet blockValSet) {
- ObjectAggregationResultHolder resultHolder = (ObjectAggregationResultHolder) aggregationResultHolder;
int[] value = blockValSet.getIntValuesSV();
- IntOpenHashSet valueArray = new IntOpenHashSet();
+ IntOpenHashSet valueArray = new IntOpenHashSet(length);
for (int i = 0; i < length; i++) {
valueArray.add(value[i]);
}
- resultHolder.setValue(valueArray);
+ aggregationResultHolder.setValue(valueArray);
}
@Override
protected void aggregateArrayWithNull(int length, AggregationResultHolder aggregationResultHolder,
BlockValSet blockValSet, RoaringBitmap nullBitmap) {
- ObjectAggregationResultHolder resultHolder = (ObjectAggregationResultHolder) aggregationResultHolder;
int[] value = blockValSet.getIntValuesSV();
IntOpenHashSet valueArray = new IntOpenHashSet(length);
for (int i = 0; i < length; i++) {
@@ -58,17 +54,16 @@ public class ArrayAggDistinctIntFunction extends BaseArrayAggIntFunction<IntOpen
valueArray.add(value[i]);
}
}
- resultHolder.setValue(valueArray);
+ aggregationResultHolder.setValue(valueArray);
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, int value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- IntOpenHashSet groupValue = resultHolder.getResult(groupKey);
- if (groupValue == null) {
- resultHolder.setValueForKey(groupKey, new IntOpenHashSet(value));
- } else {
- groupValue.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, int value) {
+ IntOpenHashSet valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new IntOpenHashSet();
+ resultHolder.setValueForKey(groupKey, valueSet);
}
+ valueSet.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctLongFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctLongFunction.java
index 34ba9fd91b..bc35b9cb36 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctLongFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctLongFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.apache.pinot.spi.data.FieldSpec;
import org.roaringbitmap.RoaringBitmap;
@@ -59,12 +58,12 @@ public class ArrayAggDistinctLongFunction extends BaseArrayAggLongFunction<LongO
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, long value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- resultHolder.setValueForKey(groupKey, new LongOpenHashSet());
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, long value) {
+ LongOpenHashSet valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new LongOpenHashSet();
+ resultHolder.setValueForKey(groupKey, valueSet);
}
- LongOpenHashSet groupValue = resultHolder.getResult(groupKey);
- groupValue.add(value);
+ valueSet.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctStringFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctStringFunction.java
index 947fd59ba1..0a964e3f10 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctStringFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDistinctStringFunction.java
@@ -24,7 +24,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -37,7 +36,7 @@ public class ArrayAggDistinctStringFunction extends BaseArrayAggStringFunction<O
protected void aggregateArray(int length, AggregationResultHolder aggregationResultHolder, BlockValSet blockValSet) {
ObjectOpenHashSet<String> valueArray = new ObjectOpenHashSet<>(length);
String[] value = blockValSet.getStringValuesSV();
- valueArray.addAll(Arrays.asList(value).subList(0, length));
+ valueArray.addAll(Arrays.asList(value));
aggregationResultHolder.setValue(valueArray);
}
@@ -55,15 +54,12 @@ public class ArrayAggDistinctStringFunction extends BaseArrayAggStringFunction<O
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, String value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- ObjectOpenHashSet<String> valueArray = new ObjectOpenHashSet<>();
- valueArray.add(value);
- resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- ObjectOpenHashSet<String> valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, String value) {
+ ObjectOpenHashSet<String> valueSet = resultHolder.getResult(groupKey);
+ if (valueSet == null) {
+ valueSet = new ObjectOpenHashSet<>();
+ resultHolder.setValueForKey(groupKey, valueSet);
}
+ valueSet.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDoubleFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDoubleFunction.java
index 20c22592c6..78a29697be 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDoubleFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggDoubleFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -56,15 +55,12 @@ public class ArrayAggDoubleFunction extends BaseArrayAggDoubleFunction<DoubleArr
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, double value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- DoubleArrayList valueArray = new DoubleArrayList();
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, double value) {
+ DoubleArrayList valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new DoubleArrayList();
resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- DoubleArrayList valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
}
+ valueArray.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggFloatFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggFloatFunction.java
index 9c43cd1138..50cb800801 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggFloatFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggFloatFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -57,15 +56,12 @@ public class ArrayAggFloatFunction extends BaseArrayAggFloatFunction<FloatArrayL
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, float value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- FloatArrayList valueArray = new FloatArrayList();
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, float value) {
+ FloatArrayList valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new FloatArrayList();
resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- FloatArrayList valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
}
+ valueArray.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggIntFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggIntFunction.java
index 2e6764b190..45589ab739 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggIntFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggIntFunction.java
@@ -56,12 +56,12 @@ public class ArrayAggIntFunction extends BaseArrayAggIntFunction<IntArrayList> {
aggregationResultHolder.setValue(valueArray);
}
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, int value) {
- IntArrayList groupValue = groupByResultHolder.getResult(groupKey);
- if (groupValue == null) {
- groupByResultHolder.setValueForKey(groupKey, new IntArrayList(value));
- } else {
- groupValue.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, int value) {
+ IntArrayList valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new IntArrayList();
+ resultHolder.setValueForKey(groupKey, valueArray);
}
+ valueArray.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggLongFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggLongFunction.java
index 95fd5954e0..61643a5fda 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggLongFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggLongFunction.java
@@ -23,7 +23,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.apache.pinot.spi.data.FieldSpec;
import org.roaringbitmap.RoaringBitmap;
@@ -58,12 +57,12 @@ public class ArrayAggLongFunction extends BaseArrayAggLongFunction<LongArrayList
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, long value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- resultHolder.setValueForKey(groupKey, new LongArrayList());
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, long value) {
+ LongArrayList valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new LongArrayList();
+ resultHolder.setValueForKey(groupKey, valueArray);
}
- LongArrayList groupValue = resultHolder.getResult(groupKey);
- groupValue.add(value);
+ valueArray.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggStringFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggStringFunction.java
index ad13868373..5abfe51dd2 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggStringFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/ArrayAggStringFunction.java
@@ -24,7 +24,6 @@ import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.query.aggregation.AggregationResultHolder;
import org.apache.pinot.core.query.aggregation.groupby.GroupByResultHolder;
-import org.apache.pinot.core.query.aggregation.groupby.ObjectGroupByResultHolder;
import org.roaringbitmap.RoaringBitmap;
@@ -37,7 +36,7 @@ public class ArrayAggStringFunction extends BaseArrayAggStringFunction<ObjectArr
protected void aggregateArray(int length, AggregationResultHolder aggregationResultHolder, BlockValSet blockValSet) {
ObjectArrayList<String> valueArray = new ObjectArrayList<>(length);
String[] value = blockValSet.getStringValuesSV();
- valueArray.addAll(Arrays.asList(value).subList(0, length));
+ valueArray.addAll(Arrays.asList(value));
aggregationResultHolder.setValue(valueArray);
}
@@ -55,15 +54,12 @@ public class ArrayAggStringFunction extends BaseArrayAggStringFunction<ObjectArr
}
@Override
- protected void setGroupByResult(GroupByResultHolder groupByResultHolder, int groupKey, String value) {
- ObjectGroupByResultHolder resultHolder = (ObjectGroupByResultHolder) groupByResultHolder;
- if (resultHolder.getResult(groupKey) == null) {
- ObjectArrayList<String> valueArray = new ObjectArrayList<>();
- valueArray.add(value);
+ protected void setGroupByResult(GroupByResultHolder resultHolder, int groupKey, String value) {
+ ObjectArrayList<String> valueArray = resultHolder.getResult(groupKey);
+ if (valueArray == null) {
+ valueArray = new ObjectArrayList<>();
resultHolder.setValueForKey(groupKey, valueArray);
- } else {
- ObjectArrayList<String> valueArray = resultHolder.getResult(groupKey);
- valueArray.add(value);
}
+ valueArray.add(value);
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggFunction.java
index 7a90d7ef30..5f17c16a19 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/array/BaseArrayAggFunction.java
@@ -92,15 +92,14 @@ public abstract class BaseArrayAggFunction<I, F extends Comparable> extends Base
public void aggregateGroupBySV(int length, int[] groupKeyArray, GroupByResultHolder groupByResultHolder,
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
- RoaringBitmap nullBitmap = null;
if (_nullHandlingEnabled) {
- nullBitmap = blockValSet.getNullBitmap();
+ RoaringBitmap nullBitmap = blockValSet.getNullBitmap();
if (nullBitmap != null && !nullBitmap.isEmpty()) {
- aggregateArrayGroupBySV(length, groupKeyArray, groupByResultHolder, blockValSet);
+ aggregateArrayGroupBySVWithNull(length, groupKeyArray, groupByResultHolder, blockValSet, nullBitmap);
return;
}
}
- aggregateArrayGroupBySVWithNull(length, groupKeyArray, groupByResultHolder, blockValSet, nullBitmap);
+ aggregateArrayGroupBySV(length, groupKeyArray, groupByResultHolder, blockValSet);
}
protected abstract void aggregateArrayGroupBySV(int length, int[] groupKeyArray,
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
index 0cf7ebdbf8..a7de368720 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ArrayTest.java
@@ -44,6 +44,7 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
private static final String DOUBLE_COLUMN = "doubleCol";
private static final String STRING_COLUMN = "stringCol";
private static final String TIMESTAMP_COLUMN = "timestampCol";
+ private static final String GROUP_BY_COLUMN = "groupKey";
@Override
protected long getCountStarResult() {
@@ -51,7 +52,7 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
}
@Test(dataProvider = "useBothQueryEngines")
- public void testQueries(boolean useMultiStageQueryEngine)
+ public void testArrayAggQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
String query =
@@ -66,7 +67,6 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
+ "arrayAgg(timestampCol, 'TIMESTAMP') "
+ "FROM %s LIMIT %d", getTableName(), getCountStarResult());
JsonNode jsonNode = postQuery(query);
- System.out.println(jsonNode);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").size(), 1);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).size(), 7);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).size(), getCountStarResult());
@@ -74,10 +74,43 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(2).size(), getCountStarResult());
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(3).size(), getCountStarResult());
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(4).size(), getCountStarResult());
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(5).size(), getCountStarResult());
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(6).size(), getCountStarResult());
}
@Test(dataProvider = "useBothQueryEngines")
- public void testQueryWithDistinct(boolean useMultiStageQueryEngine)
+ public void testArrayAggGroupByQueries(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ String query =
+ String.format("SELECT "
+ + "arrayAgg(boolCol, 'BOOLEAN'), "
+ + "arrayAgg(intCol, 'INT'), "
+ + "arrayAgg(longCol, 'LONG'), "
+ // NOTE: FLOAT array is auto converted to DOUBLE array
+ + (useMultiStageQueryEngine ? "arrayAgg(floatCol, 'DOUBLE'), " : "arrayAgg(floatCol, 'FLOAT'), ")
+ + "arrayAgg(doubleCol, 'DOUBLE'), "
+ + "arrayAgg(stringCol, 'STRING'), "
+ + "arrayAgg(timestampCol, 'TIMESTAMP'), "
+ + "groupKey "
+ + "FROM %s "
+ + "GROUP BY groupKey "
+ + "LIMIT %d", getTableName(), getCountStarResult());
+ JsonNode jsonNode = postQuery(query);
+ System.out.println("jsonNode = " + jsonNode);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").size(), 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).size(), 8);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(1).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(2).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(3).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(4).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(5).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(6).size(), getCountStarResult() / 10);
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArrayAggDistinctQueries(boolean useMultiStageQueryEngine)
throws Exception {
setUseMultiStageQueryEngine(useMultiStageQueryEngine);
String query =
@@ -93,7 +126,6 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
+ "arrayAgg(timestampCol, 'TIMESTAMP', true) "
+ "FROM %s LIMIT %d", getTableName(), getCountStarResult());
JsonNode jsonNode = postQuery(query);
- System.out.println(jsonNode);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").size(), 1);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).size(), 7);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).size(), 2);
@@ -101,6 +133,39 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(2).size(), getCountStarResult() / 10);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(3).size(), getCountStarResult() / 10);
Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(4).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(5).size(), getCountStarResult() / 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(6).size(), getCountStarResult() / 10);
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArrayAggDistinctGroupByQueries(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ String query =
+ String.format("SELECT "
+ + "arrayAgg(boolCol, 'BOOLEAN', true), "
+ + "arrayAgg(intCol, 'INT', true), "
+ + "arrayAgg(longCol, 'LONG', true), "
+ // NOTE: FLOAT array is auto converted to DOUBLE array
+ + (useMultiStageQueryEngine ? "arrayAgg(floatCol, 'DOUBLE', true), "
+ : "arrayAgg(floatCol, 'FLOAT', true), ")
+ + "arrayAgg(doubleCol, 'DOUBLE', true), "
+ + "arrayAgg(stringCol, 'STRING', true), "
+ + "arrayAgg(timestampCol, 'TIMESTAMP', true), "
+ + "groupKey "
+ + "FROM %s "
+ + "GROUP BY groupKey "
+ + "LIMIT %d", getTableName(), getCountStarResult());
+ JsonNode jsonNode = postQuery(query);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").size(), 10);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).size(), 8);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(0).size(), 2);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(1).size(), getCountStarResult() / 100);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(2).size(), getCountStarResult() / 100);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(3).size(), getCountStarResult() / 100);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(4).size(), getCountStarResult() / 100);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(5).size(), getCountStarResult() / 100);
+ Assert.assertEquals(jsonNode.get("resultTable").get("rows").get(0).get(6).size(), getCountStarResult() / 100);
}
@Override
@@ -118,6 +183,7 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
.addSingleValueDimension(DOUBLE_COLUMN, FieldSpec.DataType.DOUBLE)
.addSingleValueDimension(STRING_COLUMN, FieldSpec.DataType.STRING)
.addSingleValueDimension(TIMESTAMP_COLUMN, FieldSpec.DataType.TIMESTAMP)
+ .addSingleValueDimension(GROUP_BY_COLUMN, FieldSpec.DataType.STRING)
.build();
}
@@ -144,6 +210,9 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
null, null),
new org.apache.avro.Schema.Field(TIMESTAMP_COLUMN,
org.apache.avro.Schema.create(org.apache.avro.Schema.Type.LONG),
+ null, null),
+ new org.apache.avro.Schema.Field(GROUP_BY_COLUMN,
+ org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING),
null, null)
));
@@ -154,16 +223,18 @@ public class ArrayTest extends CustomDataQueryClusterIntegrationTest {
fileWriter.create(avroSchema, avroFile);
for (int i = 0; i < getCountStarResult(); i++) {
// add avro record to file
+ int finalI = i;
fileWriter.append(recordCache.get((int) (i % (getCountStarResult() / 10)), () -> {
// create avro record
GenericData.Record record = new GenericData.Record(avroSchema);
record.put(BOOLEAN_COLUMN, RANDOM.nextBoolean());
- record.put(INT_COLUMN, RANDOM.nextInt());
- record.put(LONG_COLUMN, RANDOM.nextLong());
- record.put(FLOAT_COLUMN, RANDOM.nextFloat());
- record.put(DOUBLE_COLUMN, RANDOM.nextDouble());
+ record.put(INT_COLUMN, finalI);
+ record.put(LONG_COLUMN, finalI);
+ record.put(FLOAT_COLUMN, finalI + RANDOM.nextFloat());
+ record.put(DOUBLE_COLUMN, finalI + RANDOM.nextDouble());
record.put(STRING_COLUMN, RandomStringUtils.random(RANDOM.nextInt(100)));
- record.put(TIMESTAMP_COLUMN, RANDOM.nextLong());
+ record.put(TIMESTAMP_COLUMN, finalI);
+ record.put(GROUP_BY_COLUMN, String.valueOf(finalI % 10));
return record;
}
));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org