You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/28 05:31:44 UTC
[doris] branch branch-1.1-lts updated: [cherry-pick][fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931) (#15425)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new e11da0e7ab [cherry-pick][fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931) (#15425)
e11da0e7ab is described below
commit e11da0e7ab2542fe0dabf22bbdc4b2dc390e6315
Author: ZenoYang <co...@qq.com>
AuthorDate: Wed Dec 28 13:31:38 2022 +0800
[cherry-pick][fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931) (#15425)
When using hll_union aggregate function, PREAGGREGATION is always OFF and Rollup cannot be hit.
---
.../org/apache/doris/analysis/AnalyticExpr.java | 6 ++----
.../apache/doris/analysis/FunctionCallExpr.java | 8 ++++----
.../apache/doris/catalog/AggregateFunction.java | 10 ++++++----
.../java/org/apache/doris/catalog/FunctionSet.java | 4 ++++
.../apache/doris/planner/SingleNodePlanner.java | 13 ++++++++----
.../rewrite/mvrewrite/HLLHashToSlotRefRule.java | 9 +++++----
.../apache/doris/rewrite/mvrewrite/NDVToHll.java | 3 ++-
.../org/apache/doris/planner/QueryPlanTest.java | 23 ++++++++++++++++++++++
8 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
index 180bcc76df..35357255a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.AnalyticWindow.Boundary;
import org.apache.doris.analysis.AnalyticWindow.BoundaryType;
import org.apache.doris.catalog.AggregateFunction;
import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
@@ -91,9 +92,6 @@ public class AnalyticExpr extends Expr {
// additional null handling in the backend.
public static String FIRST_VALUE_REWRITE = "FIRST_VALUE_REWRITE";
- // The function of HLL_UNION_AGG can't be used with a window by now.
- public static String HLL_UNION_AGG = "HLL_UNION_AGG";
-
public AnalyticExpr(FunctionCallExpr fnCall, List<Expr> partitionExprs,
List<OrderByElement> orderByElements, AnalyticWindow window) {
Preconditions.checkNotNull(fnCall);
@@ -233,7 +231,7 @@ public class AnalyticExpr extends Expr {
return false;
}
- return fn.functionName().equalsIgnoreCase(HLL_UNION_AGG);
+ return fn.functionName().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index bb1b32261f..8c92dc3cb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -552,9 +552,9 @@ public class FunctionCallExpr extends Expr {
return;
}
- if ((fnName.getFunction().equalsIgnoreCase("HLL_UNION_AGG")
- || fnName.getFunction().equalsIgnoreCase("HLL_CARDINALITY")
- || fnName.getFunction().equalsIgnoreCase("HLL_RAW_AGG"))
+ if ((fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_CARDINALITY)
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_RAW_AGG))
&& !arg.type.isHllType()) {
throw new AnalysisException(
"HLL_UNION_AGG, HLL_RAW_AGG and HLL_CARDINALITY's params must be hll column");
@@ -566,7 +566,7 @@ public class FunctionCallExpr extends Expr {
} else if (fnName.getFunction().equalsIgnoreCase("DISTINCT_PC")
|| fnName.getFunction().equalsIgnoreCase("DISTINCT_PCSA")
|| fnName.getFunction().equalsIgnoreCase("NDV")
- || fnName.getFunction().equalsIgnoreCase("HLL_UNION_AGG")) {
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)) {
fnParams.setIsDistinct(false);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
index 8a8de8214f..330cb0114d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
@@ -49,10 +49,12 @@ public class AggregateFunction extends Function {
private static final Logger LOG = LogManager.getLogger(AggregateFunction.class);
public static ImmutableSet<String> NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET = ImmutableSet.of("row_number", "rank",
- "dense_rank", "multi_distinct_count", "multi_distinct_sum", "hll_union_agg", "hll_union", "bitmap_union",
- "bitmap_intersect", "orthogonal_bitmap_intersect", "orthogonal_bitmap_intersect_count", "intersect_count",
- "orthogonal_bitmap_union_count", FunctionSet.COUNT, "approx_count_distinct", "ndv",
- FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize", FunctionSet.WINDOW_FUNNEL);
+ "dense_rank", "multi_distinct_count", "multi_distinct_sum", FunctionSet.HLL_UNION_AGG,
+ FunctionSet.HLL_UNION, FunctionSet.BITMAP_UNION, FunctionSet.BITMAP_INTERSECT,
+ FunctionSet.ORTHOGONAL_BITMAP_INTERSECT, FunctionSet.ORTHOGONAL_BITMAP_INTERSECT_COUNT,
+ FunctionSet.INTERSECT_COUNT, FunctionSet.ORTHOGONAL_BITMAP_UNION_COUNT, FunctionSet.COUNT,
+ "approx_count_distinct", "ndv", FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT,
+ "ndv_no_finalize", FunctionSet.WINDOW_FUNNEL);
public static ImmutableSet<String> ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
ImmutableSet.of("stddev_samp", "variance_samp", "var_samp", "percentile_approx");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index aa9020303a..ca2c0f0df7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -582,8 +582,12 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
.put(Type.DECIMALV2,
"33decimalv2_knuth_var_pop_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE")
.build();
+
public static final String HLL_HASH = "hll_hash";
public static final String HLL_UNION = "hll_union";
+ public static final String HLL_UNION_AGG = "hll_union_agg";
+ public static final String HLL_RAW_AGG = "hll_raw_agg";
+ public static final String HLL_CARDINALITY = "hll_cardinality";
private static final Map<Type, String> HLL_UPDATE_SYMBOL =
ImmutableMap.<Type, String>builder()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 3f6c569d68..26b3b8effb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -572,10 +572,15 @@ public class SingleNodePlanner {
returnColumnValidate = false;
break;
}
- } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("HLL_UNION_AGG")) {
- // do nothing
- } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("HLL_RAW_AGG")) {
- // do nothing
+ } else if (aggExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
+ || aggExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)
+ || aggExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION)) {
+ if (col.getAggregationType() != AggregateType.HLL_UNION) {
+ turnOffReason =
+ "Aggregate Operator not match: HLL_UNION <--> " + col.getAggregationType();
+ returnColumnValidate = false;
+ break;
+ }
} else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("NDV")) {
if ((!col.isKey())) {
turnOffReason = "NDV function with non-key column: " + col.getName();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
index 529538eb33..5ee0efbbac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
@@ -26,6 +26,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
@@ -57,16 +58,16 @@ public class HLLHashToSlotRefRule implements ExprRewriteRule {
}
FunctionCallExpr fnExpr = (FunctionCallExpr) expr;
String fnNameString = fnExpr.getFnName().getFunction();
- if (!fnNameString.equalsIgnoreCase("hll_union")
- && !fnNameString.equalsIgnoreCase("hll_raw_agg")
- && !fnNameString.equalsIgnoreCase("hll_union_agg")) {
+ if (!fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION)
+ && !fnNameString.equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)
+ && !fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)) {
return expr;
}
if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) {
return expr;
}
FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0);
- if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) {
+ if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) {
return expr;
}
if (child0FnExpr.getChild(0) instanceof SlotRef) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
index b6e77ae840..4c8c165580 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
@@ -93,7 +94,7 @@ public class NDVToHll implements ExprRewriteRule{
SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName());
List<Expr> newFnParams = Lists.newArrayList();
newFnParams.add(mvSlotRef);
- FunctionCallExpr result = new FunctionCallExpr("hll_union_agg", newFnParams);
+ FunctionCallExpr result = new FunctionCallExpr(FunctionSet.HLL_UNION_AGG, newFnParams);
result.analyzeNoThrow(analyzer);
return result;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index d222e7536c..79f8389018 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -2132,4 +2132,27 @@ public class QueryPlanTest {
String explainString2 = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryTableStr);
Assert.assertTrue(explainString2.contains("PREAGGREGATION: ON"));
}
+
+ @Test
+ public void testPreaggregationOfHllUnion() throws Exception {
+ connectContext.setDatabase("default_cluster:test");
+ createTable("create table test.test_hll(\n"
+ + " dt date,\n"
+ + " id int,\n"
+ + " name char(10),\n"
+ + " province char(10),\n"
+ + " os char(10),\n"
+ + " pv hll hll_union\n"
+ + ")\n"
+ + "Aggregate KEY (dt,id,name,province,os)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "PROPERTIES(\n"
+ + " \"replication_num\" = \"1\",\n"
+ + " \"in_memory\"=\"false\"\n"
+ + ");");
+
+ String queryBaseTableStr = "explain select dt, hll_union(pv) from test.test_hll group by dt";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryBaseTableStr);
+ Assert.assertTrue(explainString.contains("PREAGGREGATION: ON"));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org