You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/23 05:22:34 UTC
(doris) 03/09: [Feature](materialized-view) support predicate apprear both on key and value mv column (#30215)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1e74ad3f3b88b3eeaa145cca20c52dfe7472f124
Author: Pxl <px...@qq.com>
AuthorDate: Tue Jan 23 10:39:18 2024 +0800
[Feature](materialized-view) support predicate apprear both on key and value mv column (#30215)
support predicate apprear both on key and value mv column
---
.../mv/AbstractSelectMaterializedIndexRule.java | 3 +-
.../mv/SelectMaterializedIndexWithAggregate.java | 96 ++++++----------------
.../testAggQueryOnAggMV1/testAggQueryOnAggMV1.out | 6 ++
.../testAggQueryOnAggMV1.groovy | 14 +++-
4 files changed, 46 insertions(+), 73 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
index 02c033a0362..7cd63642b39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
@@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
@@ -160,7 +161,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
return true;
}
if (expression.children().isEmpty()) {
- return false;
+ return expression instanceof VirtualSlotReference;
}
for (Expression child : expression.children()) {
if (child instanceof Literal) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
index c0f70caf851..a254e8d67d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
@@ -204,7 +204,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Optional.of(project)),
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer()),
- collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
+ collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project))
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
@@ -251,7 +251,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.collect(Collectors.toSet());
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
- agg.getExpressions(), project.getProjects()))
+ agg.getExpressions(), Optional.of(project)))
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
@@ -320,9 +320,9 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalOlapScan scan = project.child();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
- agg.getExpressions(), project.getProjects()))
+ agg.getExpressions(), Optional.of(project)))
.addAll(collectRequireExprWithAggAndProject(
- filter.getExpressions(), project.getProjects()))
+ filter.getExpressions(), Optional.of(project)))
.build();
SelectResult result = select(
scan,
@@ -482,7 +482,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Optional.of(project)),
ExpressionUtils.replace(nonVirtualGroupByExprs(agg),
project.getAliasToProducer()),
- collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
+ collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project))
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
@@ -536,7 +536,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.collect(Collectors.toSet());
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
- agg.getExpressions(), project.getProjects()))
+ agg.getExpressions(), Optional.of(project)))
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
@@ -600,9 +600,9 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalOlapScan scan = project.child();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
- agg.getExpressions(), project.getProjects()))
+ agg.getExpressions(), Optional.of(project)))
.addAll(collectRequireExprWithAggAndProject(
- filter.getExpressions(), project.getProjects()))
+ filter.getExpressions(), Optional.of(project)))
.build();
SelectResult result = select(
scan,
@@ -1084,41 +1084,10 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
ExprRewriteMap exprRewriteMap = new ExprRewriteMap();
RewriteContext context = new RewriteContext(new CheckContext(scan, index.getId()), exprRewriteMap);
aggregateFunctions.forEach(aggFun -> AggFuncRewriter.rewrite(aggFun, context));
-
- // has rewritten agg functions
- Map<Slot, Slot> slotMap = exprRewriteMap.slotMap;
- // Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions.
- // For example: we have a duplicated-type table t(c1, c2) and a materialized index that has
- // a bitmap_union column `mv_bitmap_union_c2` for the column c2.
- // The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized
- // index because we have a filter `c2 > 0` for the aggregated column c2.
- Set<Slot> slotsToReplace = slotMap.keySet();
- Set<String> indexConjuncts;
- try {
- indexConjuncts = PlanNode
- .splitAndCompoundPredicateToConjuncts(context.checkContext.getMeta().getWhereClause()).stream()
- .map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet());
- } catch (Exception e) {
- return new AggRewriteResult(index, false, null, null);
- }
- if (isInputSlotsContainsNone(
- predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()),
- slotsToReplace)) {
- ImmutableSet<Slot> newRequiredSlots = requiredScanOutput.stream()
- .map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap)).collect(ImmutableSet.toImmutableSet());
- return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap);
- }
-
- return new AggRewriteResult(index, false, null, null);
+ return new AggRewriteResult(index, true, requiredScanOutput, exprRewriteMap);
}
private static class ExprRewriteMap {
-
- /**
- * Replace map for scan output slot.
- */
- public final Map<Slot, Slot> slotMap;
-
/**
* Replace map for expressions in project.
*/
@@ -1131,13 +1100,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
private Map<String, AggregateFunction> aggFuncStrMap;
public ExprRewriteMap() {
- this.slotMap = Maps.newHashMap();
this.projectExprMap = Maps.newHashMap();
this.aggFuncMap = Maps.newHashMap();
}
public boolean isEmpty() {
- return slotMap.isEmpty();
+ return aggFuncMap.isEmpty();
}
private void buildStrMap() {
@@ -1238,7 +1206,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.orElseThrow(() -> new AnalysisException(
"cannot find bitmap union slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), bitmapUnionSlot);
BitmapUnionCount bitmapUnionCount = new BitmapUnionCount(bitmapUnionSlot);
context.exprRewriteMap.aggFuncMap.put(count, bitmapUnionCount);
@@ -1269,9 +1236,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName()))).findFirst()
.orElseThrow(() -> new AnalysisException("cannot find count slot when select mv"));
- if (child instanceof Slot) {
- context.exprRewriteMap.slotMap.put((Slot) child, countSlot);
- }
context.exprRewriteMap.projectExprMap.put(child, countSlot);
Sum sum = new Sum(countSlot);
context.exprRewriteMap.aggFuncMap.put(count, sum);
@@ -1308,7 +1272,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.findFirst().orElseThrow(
() -> new AnalysisException("cannot find bitmap union slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot);
context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionSlot);
BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
@@ -1328,9 +1291,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
.orElseThrow(() -> new AnalysisException("cannot find bitmap union slot when select mv"));
- if (child instanceof Slot) {
- context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionSlot);
- }
context.exprRewriteMap.projectExprMap.put(child, bitmapUnionSlot);
BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
@@ -1370,7 +1330,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.orElseThrow(() -> new AnalysisException(
"cannot find bitmap union count slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionCountSlot);
BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
@@ -1390,9 +1349,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream().filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst().orElseThrow(
() -> new AnalysisException("cannot find bitmap union count slot when select mv"));
- if (child instanceof Slot) {
- context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionCountSlot);
- }
context.exprRewriteMap.projectExprMap.put(child, bitmapUnionCountSlot);
BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
@@ -1429,7 +1385,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.orElseThrow(() -> new AnalysisException(
"cannot find hll union slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot);
HllUnion newHllUnion = new HllUnion(hllUnionSlot);
context.exprRewriteMap.aggFuncMap.put(hllUnion, newHllUnion);
@@ -1467,7 +1422,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.orElseThrow(() -> new AnalysisException(
"cannot find hll union slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot);
HllUnionAgg newHllUnionAgg = new HllUnionAgg(hllUnionSlot);
context.exprRewriteMap.aggFuncMap.put(hllUnionAgg, newHllUnionAgg);
@@ -1506,7 +1460,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.orElseThrow(() -> new AnalysisException(
"cannot find hll union slot when select mv"));
- context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), hllUnionSlot);
HllUnionAgg hllUnionAgg = new HllUnionAgg(hllUnionSlot);
context.exprRewriteMap.aggFuncMap.put(ndv, hllUnionAgg);
@@ -1531,9 +1484,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Slot sumSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index).stream()
.filter(s -> sumColumn.equalsIgnoreCase(normalizeName(s.getName()))).findFirst()
.orElseThrow(() -> new AnalysisException("cannot find sum slot when select mv"));
- for (Slot slot : sum.child().getInputSlots()) {
- context.exprRewriteMap.slotMap.put(slot, sumSlot);
- }
context.exprRewriteMap.projectExprMap.put(sum.child(), sumSlot);
Sum newSum = new Sum(sumSlot);
context.exprRewriteMap.aggFuncMap.put(sum, newSum);
@@ -1560,7 +1510,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Set<Slot> slots = aggregateFunction.collect(SlotReference.class::isInstance);
for (Slot slot : slots) {
- context.exprRewriteMap.slotMap.put(slot, aggStateSlot);
context.exprRewriteMap.projectExprMap.put(slot, aggStateSlot);
}
@@ -1675,8 +1624,13 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
* +--LogicalOlapScan()
* t -> abs(k1#0) + 1
*/
- private Set<Expression> collectRequireExprWithAggAndProject(
- List<? extends Expression> aggExpressions, List<NamedExpression> projectExpressions) {
+ private Set<Expression> collectRequireExprWithAggAndProject(List<? extends Expression> aggExpressions,
+ Optional<LogicalProject<?>> project) {
+ List<NamedExpression> projectExpressions = project.isPresent() ? project.get().getProjects() : null;
+ if (projectExpressions == null) {
+ return aggExpressions.stream().collect(ImmutableSet.toImmutableSet());
+ }
+ Optional<Map<Slot, Expression>> slotToProducerOpt = project.map(Project::getAliasToProducer);
Map<ExprId, Expression> exprIdToExpression = projectExpressions.stream()
.collect(Collectors.toMap(NamedExpression::getExprId, e -> {
if (e instanceof Alias) {
@@ -1684,13 +1638,13 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
}
return e;
}));
- return aggExpressions.stream()
- .map(e -> {
- if ((e instanceof NamedExpression)
- && exprIdToExpression.containsKey(((NamedExpression) e).getExprId())) {
- return exprIdToExpression.get(((NamedExpression) e).getExprId());
- }
- return e;
- }).collect(ImmutableSet.toImmutableSet());
+ return aggExpressions.stream().map(e -> {
+ if ((e instanceof NamedExpression) && exprIdToExpression.containsKey(((NamedExpression) e).getExprId())) {
+ return exprIdToExpression.get(((NamedExpression) e).getExprId());
+ }
+ return e;
+ }).map(e -> {
+ return slotToProducerOpt.map(slotToExpressions -> ExpressionUtils.replace(e, slotToExpressions)).orElse(e);
+ }).collect(ImmutableSet.toImmutableSet());
}
}
diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out
index cd7da97e1a7..f37d3ef54f3 100644
--- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out
+++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out
@@ -23,3 +23,9 @@
2 2
3 3
+-- !select_mv --
+1 2
+
+-- !select_mv --
+1 2 1
+
diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
index 1b7bc10a39c..9b3e8ba371e 100644
--- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
+++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy
@@ -37,7 +37,7 @@ suite ("testAggQueryOnAggMV1") {
sql """insert into emps values("2020-01-03",3,"c",3,3,3);"""
- createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno ;")
+ createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;")
createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;")
createMV("create materialized view emps_mv_if as select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno;")
@@ -73,4 +73,16 @@ suite ("testAggQueryOnAggMV1") {
contains "(emps_mv_if)"
}
qt_select_mv "select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno order by deptno;"
+
+ explain {
+ sql("select deptno, count(deptno) from emps where deptno=1 group by deptno order by deptno;")
+ contains "(emps_mv_count_key)"
+ }
+ qt_select_mv "select deptno, count(deptno) from emps where deptno=1 group by deptno order by deptno;"
+
+ explain {
+ sql("select deptno, sum(salary), max(commission) from emps where salary=1 group by deptno order by deptno;")
+ contains "(emps)"
+ }
+ qt_select_mv "select deptno, sum(salary), max(commission) from emps where salary=1 group by deptno order by deptno;"
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org