You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/18 04:45:43 UTC
[doris] branch master updated: [fix](Nereids & planner) fix 3 plan errors about mv selection (#20939)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 905ba3fa86 [fix](Nereids & planner) fix 3 plan errors about mv selection (#20939)
905ba3fa86 is described below
commit 905ba3fa86faf06a9aaaef27213c50975d6e77b6
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Sun Jun 18 12:45:34 2023 +0800
[fix](Nereids & planner) fix 3 plan errors about mv selection (#20939)
fix 3 issues about mv selection:
1. in legacy planner, should not consider aux expr when do mv selection
2. in Nereids, should not hit mv when agg function on value column is distinct
3. select mv cannot rewrite agg function in on table with column name in uppercase
---
.../mv/AbstractSelectMaterializedIndexRule.java | 27 ++++----
.../mv/SelectMaterializedIndexWithAggregate.java | 81 ++++++++++++----------
.../doris/planner/MaterializedViewSelector.java | 18 ++---
.../apache/doris/planner/SingleNodePlanner.java | 15 ++--
.../rules/rewrite/mv/SelectMvIndexTest.java | 53 ++++++++++----
5 files changed, 111 insertions(+), 83 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
index 46a57271d2..7fd1e080e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
@@ -59,6 +59,7 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
@@ -340,7 +341,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
if (equalColNames.contains(column.getName())) {
matchCount++;
} else if (nonEqualColNames.contains(column.getName())) {
- // Unequivalence predicate's columns can match only first column in index.
+ // un-equivalence predicate's columns can match only first column in index.
matchCount++;
break;
} else {
@@ -414,15 +415,15 @@ public abstract class AbstractSelectMaterializedIndexRule {
/** SlotContext */
protected static class SlotContext {
+
// base index Slot to selected mv Slot
public final Map<Slot, Slot> baseSlotToMvSlot;
-
- // selected mv Slot name to mv Slot
- public final Map<String, Slot> mvNameToMvSlot;
+ // selected mv Slot name to mv Slot, we must use ImmutableSortedMap because column name could be uppercase
+ public final ImmutableSortedMap<String, Slot> mvNameToMvSlot;
public SlotContext(Map<Slot, Slot> baseSlotToMvSlot, Map<String, Slot> mvNameToMvSlot) {
this.baseSlotToMvSlot = ImmutableMap.copyOf(baseSlotToMvSlot);
- this.mvNameToMvSlot = ImmutableMap.copyOf(mvNameToMvSlot);
+ this.mvNameToMvSlot = ImmutableSortedMap.copyOf(mvNameToMvSlot, String.CASE_INSENSITIVE_ORDER);
}
}
@@ -446,7 +447,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
@Override
- public LogicalAggregate visitLogicalAggregate(LogicalAggregate agg, Void ctx) {
+ public LogicalAggregate<Plan> visitLogicalAggregate(LogicalAggregate<? extends Plan> agg, Void ctx) {
Plan child = agg.child(0).accept(this, ctx);
List<Expression> groupByExprs = agg.getGroupByExpressions();
List<Expression> newGroupByExprs = groupByExprs.stream()
@@ -462,7 +463,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
@Override
- public LogicalRepeat visitLogicalRepeat(LogicalRepeat repeat, Void ctx) {
+ public LogicalRepeat<Plan> visitLogicalRepeat(LogicalRepeat<? extends Plan> repeat, Void ctx) {
Plan child = repeat.child(0).accept(this, ctx);
List<List<Expression>> groupingSets = repeat.getGroupingSets();
ImmutableList.Builder<List<Expression>> newGroupingExprs = ImmutableList.builder();
@@ -482,7 +483,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
@Override
- public LogicalFilter visitLogicalFilter(LogicalFilter filter, Void ctx) {
+ public LogicalFilter<Plan> visitLogicalFilter(LogicalFilter<? extends Plan> filter, Void ctx) {
Plan child = filter.child(0).accept(this, ctx);
Set<Expression> newConjuncts = ImmutableSet.copyOf(ExpressionUtils.extractConjunction(
new ReplaceExpressionWithMvColumn(slotContext).replace(filter.getPredicate())));
@@ -491,7 +492,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
@Override
- public LogicalProject visitLogicalProject(LogicalProject project, Void ctx) {
+ public LogicalProject<Plan> visitLogicalProject(LogicalProject<? extends Plan> project, Void ctx) {
Plan child = project.child(0).accept(this, ctx);
List<NamedExpression> projects = project.getProjects();
List<NamedExpression> newProjects = projects.stream()
@@ -511,15 +512,15 @@ public abstract class AbstractSelectMaterializedIndexRule {
* ReplaceExpressionWithMvColumn
*/
protected static class ReplaceExpressionWithMvColumn extends DefaultExpressionRewriter<Void> {
+
// base index Slot to selected mv Slot
private final Map<Slot, Slot> baseSlotToMvSlot;
-
- // selected mv Slot name to mv Slot
- private final Map<String, Slot> mvNameToMvSlot;
+ // selected mv Slot name to mv Slot, we must use ImmutableSortedMap because column name could be uppercase
+ private final ImmutableSortedMap<String, Slot> mvNameToMvSlot;
public ReplaceExpressionWithMvColumn(SlotContext slotContext) {
this.baseSlotToMvSlot = ImmutableMap.copyOf(slotContext.baseSlotToMvSlot);
- this.mvNameToMvSlot = ImmutableMap.copyOf(slotContext.mvNameToMvSlot);
+ this.mvNameToMvSlot = ImmutableSortedMap.copyOf(slotContext.mvNameToMvSlot, String.CASE_INSENSITIVE_ORDER);
}
public Expression replace(Expression expression) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
index 94348b7156..6c04521e79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
@@ -111,7 +111,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
return new ReplaceExpressions(slotContext)
.replace(agg.withChildren(mvPlan), mvPlan);
} else {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -155,12 +155,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(filter.withChildren(mvPlan)), mvPlan));
} else {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -200,7 +200,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -213,7 +213,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId));
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -258,7 +258,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -272,7 +272,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
filter.withChildren(mvPlan));
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -315,7 +315,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -329,7 +329,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalProject<Plan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan);
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -361,7 +361,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -369,7 +369,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan)
), mvPlan));
} else {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -415,7 +415,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -424,7 +424,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
filter.withChildren(mvPlan)
)), mvPlan));
} else {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -467,7 +467,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -483,7 +483,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
mvPlan);
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -532,7 +532,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -551,7 +551,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
filter.withChildren(mvPlan));
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -598,7 +598,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
@@ -617,7 +617,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId));
- return new LogicalProject(
+ return new LogicalProject<>(
generateProjectsAlias(agg.getOutput(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
@@ -955,6 +955,10 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
} else if (ctx.valueNameToColumn.containsKey(childNameWithFuncName)) {
AggregateType aggType = ctx.valueNameToColumn.get(childNameWithFuncName).getAggregationType();
if (aggType == matchingAggType) {
+ if (aggFunc.isDistinct()) {
+ return PreAggStatus.off(
+ String.format("Aggregate function %s is distinct aggregation", aggFunc.toSql()));
+ }
return PreAggStatus.on();
} else {
return PreAggStatus.off(String.format("Aggregate operator don't match, aggregate function: %s"
@@ -968,14 +972,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
}
private static class CheckContext {
- public final LogicalOlapScan scan;
+ public final LogicalOlapScan scan;
+ public final long index;
public final Map<String, Column> keyNameToColumn;
-
public final Map<String, Column> valueNameToColumn;
- public final long index;
-
public CheckContext(LogicalOlapScan scan, long indexId) {
this.scan = scan;
// map<is_key, map<column_name, column>>
@@ -1073,8 +1075,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized
// index because we have a filter `c2 > 0` for the aggregated column c2.
Set<Slot> slotsToReplace = slotMap.keySet();
- if (!isInputSlotsContainsAny(ImmutableList.copyOf(predicates), slotsToReplace)
- && !isInputSlotsContainsAny(groupingExprs, slotsToReplace)) {
+ if (isInputSlotsContainsNone(ImmutableList.copyOf(predicates), slotsToReplace)
+ && isInputSlotsContainsNone(groupingExprs, slotsToReplace)) {
ImmutableSet<Slot> newRequiredSlots = requiredScanOutput.stream()
.map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap))
.collect(ImmutableSet.toImmutableSet());
@@ -1129,9 +1131,9 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
}
}
- private boolean isInputSlotsContainsAny(List<Expression> expressions, Set<Slot> slotsToCheck) {
+ private boolean isInputSlotsContainsNone(List<Expression> expressions, Set<Slot> slotsToCheck) {
Set<Slot> inputSlotSet = ExpressionUtils.getInputSlotSet(expressions);
- return !Sets.intersection(inputSlotSet, slotsToCheck).isEmpty();
+ return Sets.intersection(inputSlotSet, slotsToCheck).isEmpty();
}
private static class RewriteContext {
@@ -1147,8 +1149,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
private static class AggFuncRewriter extends DefaultExpressionRewriter<RewriteContext> {
public static final AggFuncRewriter INSTANCE = new AggFuncRewriter();
- private static Expression rewrite(Expression expr, RewriteContext context) {
- return expr.accept(INSTANCE, context);
+ private static void rewrite(Expression expr, RewriteContext context) {
+ expr.accept(INSTANCE, context);
}
/**
@@ -1176,7 +1178,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find bitmap union slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), bitmapUnionSlot);
@@ -1203,7 +1206,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find count slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), countSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), countSlot);
@@ -1239,7 +1243,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find bitmap union count slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionCountSlot);
@@ -1266,7 +1271,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find bitmap union count slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(bitmapHash, bitmapUnionCountSlot);
@@ -1301,7 +1307,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find hll union slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot);
@@ -1336,7 +1343,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find hll union slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot);
@@ -1371,7 +1379,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
- .get();
+ .orElseThrow(() -> new AnalysisException(
+ "cannot find hll union slot when select mv"));
context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), hllUnionSlot);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
index a86b406d5c..d9c69111ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
@@ -157,18 +157,15 @@ public class MaterializedViewSelector {
}
}
- // Step2: check all columns in compensating predicates are available in the view
- // output
+ // Step2: check all columns in compensating predicates are available in the view output
checkCompensatingPredicates(columnNamesInPredicates.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
scanNode.getTupleId());
- // Step3: group by list in query is the subset of group by list in view or view
- // contains no aggregation
+ // Step3: group by list in query is the subset of group by list in view or view contains no aggregation
checkGrouping(table, columnNamesInGrouping.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
scanNode.getTupleId());
// Step4: aggregation functions are available in the view output
checkAggregationFunction(table, aggColumnsInQuery.get(tableId), candidateIndexIdToMeta, scanNode.getTupleId());
- // Step5: columns required to compute output expr are available in the view
- // output
+ // Step5: columns required to compute output expr are available in the view output
checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
scanNode.getTupleId());
// Step6: if table type is aggregate and the candidateIndexIdToSchema is empty,
@@ -654,7 +651,7 @@ public class MaterializedViewSelector {
}
// Step4: compute the output column
- // ISSUE-3174: all of columns which belong to top tuple should be considered in
+ // ISSUE-3174: all columns which belong to top tuple should be considered in
// selector.
List<TupleId> tupleIds = selectStmt.getTableRefIdsWithoutInlineView();
for (TupleId tupleId : tupleIds) {
@@ -664,12 +661,7 @@ public class MaterializedViewSelector {
}
private void addAggColumnInQuery(Long tableId, FunctionCallExpr fnExpr) {
- Set<FunctionCallExpr> aggColumns = aggColumnsInQuery.get(tableId);
- if (aggColumns == null) {
- aggColumns = Sets.newHashSet();
- aggColumnsInQuery.put(tableId, aggColumns);
- }
- aggColumns.add(fnExpr);
+ aggColumnsInQuery.computeIfAbsent(tableId, k -> Sets.newHashSet()).add(fnExpr);
}
private boolean aggFunctionsMatchAggColumns(Set<FunctionCallExpr> queryExprList,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 2c11c178e0..01de183592 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -615,7 +615,9 @@ public class SingleNodePlanner {
List<SlotId> conjunctSlotIds = Lists.newArrayList();
if (allConjuncts != null) {
for (Expr conjunct : allConjuncts) {
- conjunct.getIds(null, conjunctSlotIds);
+ if (!conjunct.isAuxExpr()) {
+ conjunct.getIds(null, conjunctSlotIds);
+ }
}
for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) {
if (!slot.getColumn().isKey()) {
@@ -1330,11 +1332,10 @@ public class SingleNodePlanner {
return root;
}
- public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer)
- throws UserException, MVSelectFailedException {
+ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) throws UserException {
boolean selectFailed = false;
boolean haveError = false;
- String errorMsg = "select fail reason: ";
+ StringBuilder errorMsg = new StringBuilder("select fail reason: ");
if (queryStmt instanceof SelectStmt) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
Set<TupleId> disableTuplesMVRewriter = Sets.newHashSet();
@@ -1388,9 +1389,9 @@ public class SingleNodePlanner {
}
} catch (Exception e) {
if (haveError) {
- errorMsg += ",";
+ errorMsg.append(",");
}
- errorMsg += e.getMessage();
+ errorMsg.append(e.getMessage());
haveError = true;
tupleSelectFailed = true;
}
@@ -1409,7 +1410,7 @@ public class SingleNodePlanner {
}
}
if (haveError) {
- throw new MVSelectFailedException(errorMsg);
+ throw new MVSelectFailedException(errorMsg.toString());
}
return selectFailed;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java
index a6f0a9afd0..0361c4f2cc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java
@@ -79,8 +79,9 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP
@BeforeEach
void before() throws Exception {
+ // NOTICE: make COMMISSION as uppercase to test select mv with uppercase column name.
createTable("create table " + HR_DB_NAME + "." + EMPS_TABLE_NAME + " (time_col date, empid int, "
- + "name varchar, deptno int, salary int, commission int) partition by range (time_col) "
+ + "name varchar, deptno int, salary int, COMMISSION int) partition by range (time_col) "
+ "(partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3"
+ " properties('replication_num' = '1');");
@@ -147,16 +148,14 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP
testMv(query2, EMPS_MV_NAME);
}
- // @Test
- // public void testProjectionMV4() throws Exception {
- // String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno, salary from "
- // + EMPS_TABLE_NAME + ";";
- // String query1 = "select name from " + EMPS_TABLE_NAME + " where deptno > 30 and salary > 3000;";
- // createMv(createMVSql);
- // testMv(query1, EMPS_MV_NAME);
- // String query2 = "select empid from " + EMPS_TABLE_NAME + " where deptno > 30 and empid > 10;";
- // dorisAssert.query(query2).explainWithout(QUERY_USE_EMPS_MV);
- // }
+ @Test
+ public void testProjectionMV4() throws Exception {
+ String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno, salary from "
+ + EMPS_TABLE_NAME + ";";
+ String query1 = "select name from " + EMPS_TABLE_NAME + " where deptno > 30 and salary > 3000;";
+ createMv(createMVSql);
+ testMv(query1, EMPS_MV_NAME);
+ }
@Test
public void testUnionQueryOnProjectionMV() throws Exception {
@@ -272,8 +271,8 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP
/**
* Query with cube and arithmetic expr
- * TODO: enable this when group by cube is supported.
*/
+ @Test
public void testAggQueryOnAggMV9() throws Exception {
String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno, commission;";
@@ -301,14 +300,40 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP
*/
@Test
public void testAggQueryOnAggMV11() throws Exception {
- String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, count(salary) "
+ String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno;";
- String query = "select deptno, count(salary) + count(1) from " + EMPS_TABLE_NAME
+ String query = "select deptno, sum(salary) + sum(1) from " + EMPS_TABLE_NAME
+ " group by deptno;";
createMv(createMVSql);
testMv(query, EMPS_TABLE_NAME);
}
+ /**
+ * Aggregation query with distinct on value
+ */
+ @Test
+ public void testAggQueryOnAggMV12() throws Exception {
+ String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) "
+ + "from " + EMPS_TABLE_NAME + " group by deptno;";
+ String query = "select deptno, sum(distinct salary) from " + EMPS_TABLE_NAME
+ + " group by deptno;";
+ createMv(createMVSql);
+ testMv(query, EMPS_TABLE_NAME);
+ }
+
+ /**
+ * Aggregation query with distinct on key
+ */
+ @Test
+ public void testAggQueryOnAggMV13() throws Exception {
+ String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) "
+ + "from " + EMPS_TABLE_NAME + " group by deptno;";
+ String query = "select deptno, min(distinct deptno), sum(salary) from " + EMPS_TABLE_NAME
+ + " group by deptno;";
+ createMv(createMVSql);
+ testMv(query, EMPS_MV_NAME);
+ }
+
@Test
public void testAggQueryWithSetOperandOnAggMV() throws Exception {
String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, count(salary) "
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org