You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2020/07/05 15:29:11 UTC
[incubator-pinot] branch master updated: Using dictionary based aggregation operator when there is no filter/g… (#5657)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 13a0c68 Using dictionary based aggregation operator when there is no filter/g… (#5657)
13a0c68 is described below
commit 13a0c68274b2bb81e2607d4a603a09396260cc51
Author: Kishore Gopalakrishna <g....@gmail.com>
AuthorDate: Sun Jul 5 08:28:56 2020 -0700
Using dictionary based aggregation operator when there is no filter/g… (#5657)
* Using dictionary based aggregation operator when there is no filter/group by
* Fixing test cases, numEntriesScannedPostFilter has changed if there is a filter
* Addressing comments
---
.../operator/query/DictionaryBasedAggregationOperator.java | 8 ++++++++
.../apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java | 6 ++++--
.../query/aggregation/function/AggregationFunctionUtils.java | 10 ++++++++++
.../queries/InterSegmentAggregationSingleValueQueriesTest.java | 3 ++-
.../queries/InterSegmentResultTableSingleValueQueriesTest.java | 3 ++-
5 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedAggregationOperator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
index 8c66805..971a225 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedAggregationOperator.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.query;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -73,6 +74,13 @@ public class DictionaryBasedAggregationOperator extends BaseOperator<Intermediat
aggregationResults.add(
new MinMaxRangePair(dictionary.getDoubleValue(0), dictionary.getDoubleValue(dictionary.length() - 1)));
break;
+ case DISTINCTCOUNT:
+ IntOpenHashSet set = new IntOpenHashSet(128);
+ for (int dictId = 0; dictId < dictionary.length(); dictId++) {
+ set.add(dictionary.get(dictId).hashCode());
+ }
+ aggregationResults.add(set);
+ break;
default:
throw new IllegalStateException(
"Dictionary based aggregation operator does not support function type: " + aggregationFunction.getType());
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
index 3d81a51..1658a50 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
@@ -35,6 +35,7 @@ import org.apache.pinot.core.plan.MetadataBasedAggregationPlanNode;
import org.apache.pinot.core.plan.Plan;
import org.apache.pinot.core.plan.PlanNode;
import org.apache.pinot.core.plan.SelectionPlanNode;
+import org.apache.pinot.core.query.aggregation.function.AggregationFunctionUtils;
import org.apache.pinot.core.query.config.QueryExecutorConfig;
import org.apache.pinot.core.query.request.context.ExpressionContext;
import org.apache.pinot.core.query.request.context.FunctionContext;
@@ -156,7 +157,7 @@ public class InstancePlanMakerImplV2 implements PlanMaker {
/**
* Returns {@code true} if the given aggregation-only without filter QueryContext can be solved with dictionary,
* {@code false} otherwise.
- * <p>Aggregations supported: MIN, MAX, MINMAXRANGE
+ * <p>Aggregations supported: MIN, MAX, MINMAXRANGE, DISTINCTCOUNT
*/
@VisibleForTesting
static boolean isFitForDictionaryBasedPlan(QueryContext queryContext, IndexSegment indexSegment) {
@@ -164,9 +165,10 @@ public class InstancePlanMakerImplV2 implements PlanMaker {
for (ExpressionContext expression : selectExpressions) {
FunctionContext function = expression.getFunction();
String functionName = function.getFunctionName();
- if (!functionName.equals("min") && !functionName.equals("max") && !functionName.equals("minmaxrange")) {
+ if(!AggregationFunctionUtils.isFitForDictionaryBasedComputation(functionName)) {
return false;
}
+
ExpressionContext argument = function.getArguments().get(0);
if (argument.getType() != ExpressionContext.Type.IDENTIFIER) {
return false;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
index 898e2de..3822373 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtils.java
@@ -207,4 +207,14 @@ public class AggregationFunctionUtils {
BlockValSet blockValSet = transformBlock.getBlockValueSet(aggregationFunctionColumnPair.toColumnName());
return Collections.singletonMap(expression, blockValSet);
}
+
+ public static boolean isFitForDictionaryBasedComputation(String functionName) {
+ if (functionName.equalsIgnoreCase(AggregationFunctionType.MIN.name()) || //
+ functionName.equalsIgnoreCase(AggregationFunctionType.MAX.name()) || //
+ functionName.equalsIgnoreCase(AggregationFunctionType.MINMAXRANGE.name()) || //
+ functionName.equalsIgnoreCase(AggregationFunctionType.DISTINCTCOUNT.name())) {
+ return true;
+ }
+ return false;
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
index 70654d4..cd8fd77 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentAggregationSingleValueQueriesTest.java
@@ -174,7 +174,8 @@ public class InterSegmentAggregationSingleValueQueriesTest extends BaseSingleVal
String query = "SELECT DISTINCTCOUNT(column1), DISTINCTCOUNT(column3) FROM testTable";
BrokerResponseNative brokerResponse = getBrokerResponseForPqlQuery(query);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 120000L, 0L, 240000L, 120000L,
+ //without filter, we should be using dictionary for distinctcount
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 120000L, 0L, 0L, 120000L,
new String[]{"6582", "21910"});
brokerResponse = getBrokerResponseForPqlQueryWithFilter(query);
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentResultTableSingleValueQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentResultTableSingleValueQueriesTest.java
index 5f462da..d6de3af 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentResultTableSingleValueQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/InterSegmentResultTableSingleValueQueriesTest.java
@@ -347,8 +347,9 @@ public class InterSegmentResultTableSingleValueQueriesTest extends BaseSingleVal
rows = new ArrayList<>();
rows.add(new Object[]{6582, 21910});
expectedResultsSize = 1;
+ //without filter, distinctCount must be solved using dictionary. expectedNumEntriesScannedPostFilter is 0L
QueriesTestUtils
- .testInterSegmentResultTable(brokerResponse, 120000L, 0L, 240000L, 120000L, rows, expectedResultsSize,
+ .testInterSegmentResultTable(brokerResponse, 120000L, 0L, 0L, 120000L, rows, expectedResultsSize,
dataSchema);
brokerResponse = getBrokerResponseForPqlQuery(query + getFilter(), queryOptions);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org