You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/27 03:48:04 UTC
[doris] branch branch-1.2-lts updated (e4ba61ec12 -> d5936eb48c)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from e4ba61ec12 [improvment](function) add new function substring_index (#15373)
new 1a8244b534 [fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931)
new 1033d71c1b [fix](planner) set limit to nagtive value when has offset in limit (#15367)
new d5936eb48c [fix](mow-uniquekey) fix dereference to nullptr in Tablet::calc_delete_bitmap (#15375)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/olap/tablet.cpp | 2 +-
.../org/apache/doris/analysis/AnalyticExpr.java | 6 ++----
.../apache/doris/analysis/FunctionCallExpr.java | 8 ++++----
.../apache/doris/catalog/AggregateFunction.java | 12 ++++++-----
.../java/org/apache/doris/catalog/FunctionSet.java | 4 ++++
.../apache/doris/planner/DistributedPlanner.java | 6 ++++++
.../org/apache/doris/planner/ExchangeNode.java | 2 +-
.../apache/doris/planner/SingleNodePlanner.java | 13 ++++++++----
.../rewrite/mvrewrite/HLLHashToSlotRefRule.java | 9 +++++----
.../apache/doris/rewrite/mvrewrite/NDVToHll.java | 3 ++-
.../org/apache/doris/planner/QueryPlanTest.java | 23 ++++++++++++++++++++++
11 files changed, 64 insertions(+), 24 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1a8244b5340ba9843fad462f6421bfb5962e0017
Author: ZenoYang <co...@qq.com>
AuthorDate: Tue Dec 27 11:20:41 2022 +0800
[fix](planner) fix hll_union plan: Invalid Aggregate Operator: hll_union (#14931)
When using hll_union aggregate function, PREAGGREGATION is always OFF and Rollup cannot be hit.
---
.../org/apache/doris/analysis/AnalyticExpr.java | 6 ++----
.../apache/doris/analysis/FunctionCallExpr.java | 8 ++++----
.../apache/doris/catalog/AggregateFunction.java | 12 ++++++-----
.../java/org/apache/doris/catalog/FunctionSet.java | 4 ++++
.../apache/doris/planner/SingleNodePlanner.java | 13 ++++++++----
.../rewrite/mvrewrite/HLLHashToSlotRefRule.java | 9 +++++----
.../apache/doris/rewrite/mvrewrite/NDVToHll.java | 3 ++-
.../org/apache/doris/planner/QueryPlanTest.java | 23 ++++++++++++++++++++++
8 files changed, 56 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
index a9598a11f2..f5f1aa5397 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.AnalyticWindow.Boundary;
import org.apache.doris.analysis.AnalyticWindow.BoundaryType;
import org.apache.doris.catalog.AggregateFunction;
import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
@@ -96,9 +97,6 @@ public class AnalyticExpr extends Expr {
// additional null handling in the backend.
public static String FIRST_VALUE_REWRITE = "FIRST_VALUE_REWRITE";
- // The function of HLL_UNION_AGG can't be used with a window by now.
- public static String HLL_UNION_AGG = "HLL_UNION_AGG";
-
public AnalyticExpr(FunctionCallExpr fnCall, List<Expr> partitionExprs,
List<OrderByElement> orderByElements, AnalyticWindow window) {
Preconditions.checkNotNull(fnCall);
@@ -248,7 +246,7 @@ public class AnalyticExpr extends Expr {
return false;
}
- return fn.functionName().equalsIgnoreCase(HLL_UNION_AGG);
+ return fn.functionName().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG);
}
private static boolean isNTileFn(Function fn) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index d20626e4c8..59d8fbc8f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -801,9 +801,9 @@ public class FunctionCallExpr extends Expr {
}
}
- if ((fnName.getFunction().equalsIgnoreCase("HLL_UNION_AGG")
- || fnName.getFunction().equalsIgnoreCase("HLL_CARDINALITY")
- || fnName.getFunction().equalsIgnoreCase("HLL_RAW_AGG"))
+ if ((fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_CARDINALITY)
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_RAW_AGG))
&& !arg.type.isHllType()) {
throw new AnalysisException(
"HLL_UNION_AGG, HLL_RAW_AGG and HLL_CARDINALITY's params must be hll column");
@@ -815,7 +815,7 @@ public class FunctionCallExpr extends Expr {
} else if (fnName.getFunction().equalsIgnoreCase("DISTINCT_PC")
|| fnName.getFunction().equalsIgnoreCase("DISTINCT_PCSA")
|| fnName.getFunction().equalsIgnoreCase("NDV")
- || fnName.getFunction().equalsIgnoreCase("HLL_UNION_AGG")) {
+ || fnName.getFunction().equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)) {
fnParams.setIsDistinct(false);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
index 12363ed818..0729b0aa03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
@@ -47,11 +47,13 @@ public class AggregateFunction extends Function {
private static final Logger LOG = LogManager.getLogger(AggregateFunction.class);
public static ImmutableSet<String> NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET = ImmutableSet.of("row_number", "rank",
- "dense_rank", "multi_distinct_count", "multi_distinct_sum", "hll_union_agg", "hll_union", "bitmap_union",
- "bitmap_intersect", "orthogonal_bitmap_intersect", "orthogonal_bitmap_intersect_count", "intersect_count",
- "orthogonal_bitmap_union_count", FunctionSet.COUNT, "approx_count_distinct", "ndv",
- FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize", FunctionSet.WINDOW_FUNNEL,
- FunctionSet.RETENTION, FunctionSet.SEQUENCE_MATCH, FunctionSet.SEQUENCE_COUNT);
+ "dense_rank", "multi_distinct_count", "multi_distinct_sum", FunctionSet.HLL_UNION_AGG,
+ FunctionSet.HLL_UNION, FunctionSet.BITMAP_UNION, FunctionSet.BITMAP_INTERSECT,
+ FunctionSet.ORTHOGONAL_BITMAP_INTERSECT, FunctionSet.ORTHOGONAL_BITMAP_INTERSECT_COUNT,
+ FunctionSet.INTERSECT_COUNT, FunctionSet.ORTHOGONAL_BITMAP_UNION_COUNT,
+ FunctionSet.COUNT, "approx_count_distinct", "ndv", FunctionSet.BITMAP_UNION_INT,
+ FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize", FunctionSet.WINDOW_FUNNEL, FunctionSet.RETENTION,
+ FunctionSet.SEQUENCE_MATCH, FunctionSet.SEQUENCE_COUNT);
public static ImmutableSet<String> ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
ImmutableSet.of("stddev_samp", "variance_samp", "var_samp", "percentile_approx");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index af72fd6df8..e23d2afdb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -653,8 +653,12 @@ public class FunctionSet<T> {
.put(Type.MAX_DECIMALV2_TYPE,
"33decimalv2_knuth_var_pop_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE")
.build();
+
public static final String HLL_HASH = "hll_hash";
public static final String HLL_UNION = "hll_union";
+ public static final String HLL_UNION_AGG = "hll_union_agg";
+ public static final String HLL_RAW_AGG = "hll_raw_agg";
+ public static final String HLL_CARDINALITY = "hll_cardinality";
private static final Map<Type, String> HLL_UPDATE_SYMBOL =
ImmutableMap.<Type, String>builder()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 82bceeb92a..86044c13b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -738,10 +738,15 @@ public class SingleNodePlanner {
returnColumnValidate = false;
break;
}
- } else if (functionName.equalsIgnoreCase("HLL_UNION_AGG")) {
- // do nothing
- } else if (functionName.equalsIgnoreCase("HLL_RAW_AGG")) {
- // do nothing
+ } else if (functionName.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
+ || functionName.equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)
+ || functionName.equalsIgnoreCase(FunctionSet.HLL_UNION)) {
+ if (col.getAggregationType() != AggregateType.HLL_UNION) {
+ turnOffReason =
+ "Aggregate Operator not match: HLL_UNION <--> " + col.getAggregationType();
+ returnColumnValidate = false;
+ break;
+ }
} else if (functionName.equalsIgnoreCase("NDV")) {
if ((!col.isKey())) {
turnOffReason = "NDV function with non-key column: " + col.getName();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
index cf13f52cb4..34ce7a60e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
@@ -26,6 +26,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
@@ -57,16 +58,16 @@ public class HLLHashToSlotRefRule implements ExprRewriteRule {
}
FunctionCallExpr fnExpr = (FunctionCallExpr) expr;
String fnNameString = fnExpr.getFnName().getFunction();
- if (!fnNameString.equalsIgnoreCase("hll_union")
- && !fnNameString.equalsIgnoreCase("hll_raw_agg")
- && !fnNameString.equalsIgnoreCase("hll_union_agg")) {
+ if (!fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION)
+ && !fnNameString.equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)
+ && !fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)) {
return expr;
}
if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) {
return expr;
}
FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0);
- if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) {
+ if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) {
return expr;
}
if (child0FnExpr.getChild(0) instanceof SlotRef) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
index 9f20f0a70d..159073cb05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
@@ -93,7 +94,7 @@ public class NDVToHll implements ExprRewriteRule {
SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName());
List<Expr> newFnParams = Lists.newArrayList();
newFnParams.add(mvSlotRef);
- FunctionCallExpr result = new FunctionCallExpr("hll_union_agg", newFnParams);
+ FunctionCallExpr result = new FunctionCallExpr(FunctionSet.HLL_UNION_AGG, newFnParams);
result.analyzeNoThrow(analyzer);
return result;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index d790707a88..bea801b6ba 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -2196,4 +2196,27 @@ public class QueryPlanTest extends TestWithFeService {
String explainString2 = getSQLPlanOrErrorMsg(queryTableStr);
Assert.assertTrue(explainString2.contains("PREAGGREGATION: ON"));
}
+
+ @Test
+ public void testPreaggregationOfHllUnion() throws Exception {
+ connectContext.setDatabase("default_cluster:test");
+ createTable("create table test.test_hll(\n"
+ + " dt date,\n"
+ + " id int,\n"
+ + " name char(10),\n"
+ + " province char(10),\n"
+ + " os char(10),\n"
+ + " pv hll hll_union\n"
+ + ")\n"
+ + "Aggregate KEY (dt,id,name,province,os)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "PROPERTIES(\n"
+ + " \"replication_num\" = \"1\",\n"
+ + " \"in_memory\"=\"false\"\n"
+ + ");");
+
+ String queryBaseTableStr = "explain select dt, hll_union(pv) from test.test_hll group by dt";
+ String explainString = getSQLPlanOrErrorMsg(queryBaseTableStr);
+ Assert.assertTrue(explainString.contains("PREAGGREGATION: ON"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [fix](mow-uniquekey) fix dereference to nullptr in Tablet::calc_delete_bitmap (#15375)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d5936eb48cb01f7b81405b6b0b6b6d7884a71851
Author: zhengshengjun <zh...@apache.org>
AuthorDate: Tue Dec 27 11:14:25 2022 +0800
[fix](mow-uniquekey) fix dereference to nullptr in Tablet::calc_delete_bitmap (#15375)
---
be/src/olap/tablet.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index d0dd89c0bd..3722b7d079 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2046,7 +2046,7 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
}
}
- if (!specified_rowset_ids->empty()) {
+ if (specified_rowset_ids != nullptr && !specified_rowset_ids->empty()) {
auto st = lookup_row_key(*key, specified_rowset_ids, &loc,
dummy_version.first - 1);
CHECK(st.ok() || st.is_not_found() || st.is_already_exist());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [fix](planner) set limit to nagtive value when has offset in limit (#15367)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1033d71c1ba9988e8b6c879b37abce5c169bc746
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Tue Dec 27 11:16:19 2022 +0800
[fix](planner) set limit to nagtive value when has offset in limit (#15367)
---
.../src/main/java/org/apache/doris/planner/DistributedPlanner.java | 6 ++++++
fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
index 19d153f1b5..c537434cc0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
@@ -257,6 +257,12 @@ public class DistributedPlanner {
// exchange node clones the behavior of its input, aside from the conjuncts
ExchangeNode mergePlan =
new ExchangeNode(ctx.getNextNodeId(), inputFragment.getPlanRoot(), false);
+ PlanNode inputRoot = inputFragment.getPlanRoot();
+ if (inputRoot.hasOffset()) {
+ long limit = inputRoot.getOffset() + inputRoot.getLimit();
+ inputRoot.unsetLimit();
+ inputRoot.setLimit(limit);
+ }
mergePlan.setNumInstances(inputFragment.getPlanRoot().getNumInstances());
mergePlan.init(ctx.getRootAnalyzer());
Preconditions.checkState(mergePlan.hasValidStats());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
index e3cd192f8c..0629af2295 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
@@ -82,7 +82,7 @@ public class ExchangeNode extends PlanNode {
}
// Only apply the limit at the receiver if there are multiple senders.
if (inputNode.getFragment().isPartitioned()) {
- limit = inputNode.limit - inputNode.offset;
+ limit = inputNode.limit;
}
offset = inputNode.offset;
computeTupleIds();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org