You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/07/10 05:51:54 UTC
[GitHub] [doris] 924060929 commented on a diff in pull request #10659: [enhancement](nereids) make SSB works
924060929 commented on code in PR #10659:
URL: https://github.com/apache/doris/pull/10659#discussion_r917338424
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java:
##########
@@ -81,33 +74,14 @@ public void plan(StatementBase queryStmt,
PhysicalPlanTranslator physicalPlanTranslator = new PhysicalPlanTranslator();
PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext();
- physicalPlanTranslator.translatePlan(physicalPlan, planTranslatorContext);
+ PlanFragment root = physicalPlanTranslator.translatePlan(physicalPlan, planTranslatorContext);
scanNodeList = planTranslatorContext.getScanNodeList();
descTable = planTranslatorContext.getDescTable();
fragments = new ArrayList<>(planTranslatorContext.getPlanFragmentList());
- for (PlanFragment fragment : fragments) {
- fragment.finalize(queryStmt);
- }
- Collections.reverse(fragments);
- PlanFragment root = fragments.get(0);
-
- // compute output exprs
- Map<Integer, Expr> outputCandidates = Maps.newHashMap();
- List<Expr> outputExprs = Lists.newArrayList();
- for (TupleId tupleId : root.getPlanRoot().getTupleIds()) {
- TupleDescriptor tupleDescriptor = descTable.getTupleDesc(tupleId);
- for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) {
- SlotRef slotRef = new SlotRef(slotDescriptor);
- outputCandidates.put(slotDescriptor.getId().asInt(), slotRef);
- }
- }
- physicalPlan.getOutput().stream()
- .forEach(i -> outputExprs.add(planTranslatorContext.findExpr(i)));
- root.setOutputExprs(outputExprs);
- root.getPlanRoot().convertToVectoriezd();
- logicalPlanAdapter.setResultExprs(outputExprs);
+ // set output exprs
+ logicalPlanAdapter.setResultExprs(root.getOutputExprs());
Review Comment:
great refactor
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggregateDisassemble.java:
##########
@@ -17,144 +17,156 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.analysis.FunctionName;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Function;
-import org.apache.doris.catalog.Function.CompareMode;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.plans.AggPhase;
import org.apache.doris.nereids.operators.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnaryPlan;
-import com.clearspring.analytics.util.Lists;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
- * TODO: if instance count is 1, shouldn't disassemble the agg operator
* Used to generate the merge agg node for distributed execution.
- * Do this in following steps:
- * 1. clone output expr list, find all agg function
- * 2. set found agg function intermediaType
- * 3. create new child plan rooted at new local agg
- * 4. update the slot referenced by expr of merge agg
- * 5. create plan rooted at merge agg, return it.
+ * NOTICE: GLOBAL output expressions' ExprId should SAME with ORIGIN output expressions' ExprId.
+ * If we have a query: SELECT SUM(v1 * v2) + 1 FROM t GROUP BY k + 1
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(k + 1) #1, Alias(SUM(v1 * v2) + 1) #2], groupByExpr: [k + 1])
+ * +-- childPlan
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(b) #1, Alias(SUM(a) + 1) #2], groupByExpr: [b])
+ * +-- Aggregate(phase: [LOCAL], outputExpr: [SUM(v1 * v2) as a, (k + 1) as b], groupByExpr: [k + 1])
+ * +-- childPlan
Review Comment:
great comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggregateDisassemble.java:
##########
@@ -17,144 +17,156 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.analysis.FunctionName;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Function;
-import org.apache.doris.catalog.Function.CompareMode;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.plans.AggPhase;
import org.apache.doris.nereids.operators.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnaryPlan;
-import com.clearspring.analytics.util.Lists;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
- * TODO: if instance count is 1, shouldn't disassemble the agg operator
* Used to generate the merge agg node for distributed execution.
- * Do this in following steps:
- * 1. clone output expr list, find all agg function
- * 2. set found agg function intermediaType
- * 3. create new child plan rooted at new local agg
- * 4. update the slot referenced by expr of merge agg
- * 5. create plan rooted at merge agg, return it.
+ * NOTICE: GLOBAL output expressions' ExprId should SAME with ORIGIN output expressions' ExprId.
+ * If we have a query: SELECT SUM(v1 * v2) + 1 FROM t GROUP BY k + 1
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(k + 1) #1, Alias(SUM(v1 * v2) + 1) #2], groupByExpr: [k + 1])
+ * +-- childPlan
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(b) #1, Alias(SUM(a) + 1) #2], groupByExpr: [b])
+ * +-- Aggregate(phase: [LOCAL], outputExpr: [SUM(v1 * v2) as a, (k + 1) as b], groupByExpr: [k + 1])
+ * +-- childPlan
+ *
+ * TODO:
+ * 1. use different class represent different phase aggregate
+ * 2. if instance count is 1, shouldn't disassemble the agg operator
+ * 3. we need another rule to removing duplicated expressions in group by expression list
*/
public class AggregateDisassemble extends OneRewriteRuleFactory {
@Override
public Rule<Plan> build() {
return logicalAggregate().when(p -> {
- LogicalAggregate logicalAggregation = p.getOperator();
- return !logicalAggregation.isDisassembled();
+ LogicalAggregate logicalAggregate = p.getOperator();
+ return !logicalAggregate.isDisassembled();
}).thenApply(ctx -> {
- Plan plan = ctx.root;
- Operator operator = plan.getOperator();
- LogicalAggregate agg = (LogicalAggregate) operator;
- List<NamedExpression> outputExpressionList = agg.getOutputExpressionList();
- List<NamedExpression> intermediateAggExpressionList = Lists.newArrayList();
- // TODO: shouldn't extract agg function from this field.
- for (NamedExpression namedExpression : outputExpressionList) {
- namedExpression = (NamedExpression) namedExpression.clone();
- List<AggregateFunction> functionCallList =
- namedExpression.collect(org.apache.doris.catalog.AggregateFunction.class::isInstance);
- // TODO: we will have another mechanism to get corresponding stale agg func.
- for (AggregateFunction functionCall : functionCallList) {
- org.apache.doris.catalog.AggregateFunction staleAggFunc = findAggFunc(functionCall);
- Type staleIntermediateType = staleAggFunc.getIntermediateType();
- Type staleRetType = staleAggFunc.getReturnType();
- if (staleIntermediateType != null && !staleIntermediateType.equals(staleRetType)) {
- functionCall.setIntermediate(DataType.convertFromCatalogDataType(staleIntermediateType));
+ LogicalUnaryPlan<LogicalAggregate, GroupPlan> plan = ctx.root;
+ LogicalAggregate aggregate = plan.getOperator();
+ List<NamedExpression> originOutputExprs = aggregate.getOutputExpressionList();
+ List<Expression> originGroupByExprs = aggregate.getGroupByExpressionList();
+
+ // 1. generate a map from local aggregate output to global aggregate expr substitution.
+ // inputSubstitutionMap use for replacing expression in global aggregate
+ // replace rule is:
+ // a: Expression is a group by key and is a slot reference. e.g. group by k1
+ // b. Expression is a group by key and is an expression. e.g. group by k1 + 1
+ // c. Expression is an aggregate function. e.g. sum(v1) in select list
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | situation | origin expression | local output expression | expression in global aggregate |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | a | Ref(k1)#1 | Ref(k1)#1 | Ref(k1)#1 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | b | Ref(k1)#1 + 1 | A(Ref(k1)#1 + 1, key)#2 | Ref(key)#2 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | c | A(AF(v1#1), 'af')#2 | A(AF(v1#1), 'af')#3 | AF(af#3) |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // NOTICE: Ref: SlotReference, A: Alias, AF: AggregateFunction, #x: ExprId x
+ // 2. collect local aggregate output expressions and local aggregate group by expression list
Review Comment:
great comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggregateDisassemble.java:
##########
@@ -17,144 +17,156 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.analysis.FunctionName;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Function;
-import org.apache.doris.catalog.Function.CompareMode;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.plans.AggPhase;
import org.apache.doris.nereids.operators.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnaryPlan;
-import com.clearspring.analytics.util.Lists;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
- * TODO: if instance count is 1, shouldn't disassemble the agg operator
* Used to generate the merge agg node for distributed execution.
- * Do this in following steps:
- * 1. clone output expr list, find all agg function
- * 2. set found agg function intermediaType
- * 3. create new child plan rooted at new local agg
- * 4. update the slot referenced by expr of merge agg
- * 5. create plan rooted at merge agg, return it.
+ * NOTICE: GLOBAL output expressions' ExprId should SAME with ORIGIN output expressions' ExprId.
+ * If we have a query: SELECT SUM(v1 * v2) + 1 FROM t GROUP BY k + 1
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(k + 1) #1, Alias(SUM(v1 * v2) + 1) #2], groupByExpr: [k + 1])
+ * +-- childPlan
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(b) #1, Alias(SUM(a) + 1) #2], groupByExpr: [b])
+ * +-- Aggregate(phase: [LOCAL], outputExpr: [SUM(v1 * v2) as a, (k + 1) as b], groupByExpr: [k + 1])
+ * +-- childPlan
+ *
+ * TODO:
+ * 1. use different class represent different phase aggregate
+ * 2. if instance count is 1, shouldn't disassemble the agg operator
+ * 3. we need another rule to removing duplicated expressions in group by expression list
*/
public class AggregateDisassemble extends OneRewriteRuleFactory {
@Override
public Rule<Plan> build() {
return logicalAggregate().when(p -> {
- LogicalAggregate logicalAggregation = p.getOperator();
- return !logicalAggregation.isDisassembled();
+ LogicalAggregate logicalAggregate = p.getOperator();
+ return !logicalAggregate.isDisassembled();
}).thenApply(ctx -> {
- Plan plan = ctx.root;
- Operator operator = plan.getOperator();
- LogicalAggregate agg = (LogicalAggregate) operator;
- List<NamedExpression> outputExpressionList = agg.getOutputExpressionList();
- List<NamedExpression> intermediateAggExpressionList = Lists.newArrayList();
- // TODO: shouldn't extract agg function from this field.
- for (NamedExpression namedExpression : outputExpressionList) {
- namedExpression = (NamedExpression) namedExpression.clone();
- List<AggregateFunction> functionCallList =
- namedExpression.collect(org.apache.doris.catalog.AggregateFunction.class::isInstance);
- // TODO: we will have another mechanism to get corresponding stale agg func.
- for (AggregateFunction functionCall : functionCallList) {
- org.apache.doris.catalog.AggregateFunction staleAggFunc = findAggFunc(functionCall);
- Type staleIntermediateType = staleAggFunc.getIntermediateType();
- Type staleRetType = staleAggFunc.getReturnType();
- if (staleIntermediateType != null && !staleIntermediateType.equals(staleRetType)) {
- functionCall.setIntermediate(DataType.convertFromCatalogDataType(staleIntermediateType));
+ LogicalUnaryPlan<LogicalAggregate, GroupPlan> plan = ctx.root;
+ LogicalAggregate aggregate = plan.getOperator();
+ List<NamedExpression> originOutputExprs = aggregate.getOutputExpressionList();
+ List<Expression> originGroupByExprs = aggregate.getGroupByExpressionList();
+
+ // 1. generate a map from local aggregate output to global aggregate expr substitution.
+ // inputSubstitutionMap use for replacing expression in global aggregate
+ // replace rule is:
+ // a: Expression is a group by key and is a slot reference. e.g. group by k1
+ // b. Expression is a group by key and is an expression. e.g. group by k1 + 1
+ // c. Expression is an aggregate function. e.g. sum(v1) in select list
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | situation | origin expression | local output expression | expression in global aggregate |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | a | Ref(k1)#1 | Ref(k1)#1 | Ref(k1)#1 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | b | Ref(k1)#1 + 1 | A(Ref(k1)#1 + 1, key)#2 | Ref(key)#2 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | c | A(AF(v1#1), 'af')#2 | A(AF(v1#1), 'af')#3 | AF(af#3) |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // NOTICE: Ref: SlotReference, A: Alias, AF: AggregateFunction, #x: ExprId x
+ // 2. collect local aggregate output expressions and local aggregate group by expression list
+ Map<Expression, Expression> inputSubstitutionMap = Maps.newHashMap();
+ List<Expression> localGroupByExprs = aggregate.getGroupByExpressionList();
+ List<NamedExpression> localOutputExprs = Lists.newArrayList();
+ for (Expression originGroupByExpr : originGroupByExprs) {
+ if (inputSubstitutionMap.containsKey(originGroupByExpr)) {
+ continue;
+ }
+ if (originGroupByExpr instanceof SlotReference) {
+ inputSubstitutionMap.put(originGroupByExpr, originGroupByExpr);
+ localOutputExprs.add((SlotReference) originGroupByExpr);
+ } else {
+ NamedExpression localOutputExpr = new Alias<>(originGroupByExpr, originGroupByExpr.toSql());
+ inputSubstitutionMap.put(originGroupByExpr, localOutputExpr.toSlot());
+ localOutputExprs.add(localOutputExpr);
+ }
+ }
+ for (NamedExpression originOutputExpr : originOutputExprs) {
+ List<AggregateFunction> aggregateFunctions
+ = originOutputExpr.collect(AggregateFunction.class::isInstance);
+ for (AggregateFunction aggregateFunction : aggregateFunctions) {
+ if (inputSubstitutionMap.containsKey(aggregateFunction)) {
+ continue;
}
+ NamedExpression localOutputExpr = new Alias<>(aggregateFunction, aggregateFunction.toSql());
+ Expression substitutionValue = aggregateFunction.withChildren(
+ Lists.newArrayList(localOutputExpr.toSlot()));
+ inputSubstitutionMap.put(aggregateFunction, substitutionValue);
+ localOutputExprs.add(localOutputExpr);
}
- intermediateAggExpressionList.add(namedExpression);
}
- LogicalAggregate localAgg = new LogicalAggregate(
- agg.getGroupByExprList().stream().map(Expression::clone).collect(Collectors.toList()),
- intermediateAggExpressionList,
+
+ // 3. replace expression in globalOutputExprs and globalGroupByExprs
+ List<NamedExpression> globalOutputExprs = aggregate.getOutputExpressionList().stream()
+ .map(e -> ExpressionReplacer.INSTANCE.visit(e, inputSubstitutionMap))
+ .map(NamedExpression.class::cast)
+ .collect(Collectors.toList());
+ List<Expression> globalGroupByExprs = localGroupByExprs.stream()
+ .map(e -> ExpressionReplacer.INSTANCE.visit(e, inputSubstitutionMap)).collect(Collectors.toList());
+
+ // 4. generate new plan
+ LogicalAggregate localAggregate = new LogicalAggregate(
+ localGroupByExprs,
+ localOutputExprs,
true,
- AggPhase.FIRST
+ AggPhase.LOCAL
);
-
- Plan childPlan = plan(localAgg, plan.child(0));
- List<Slot> stalePlanOutputSlotList = plan.getOutput();
- List<Slot> childOutputSlotList = childPlan.getOutput();
- int childOutputSize = stalePlanOutputSlotList.size();
- Preconditions.checkState(childOutputSize == childOutputSlotList.size());
- Map<Slot, Slot> staleToNew = new HashMap<>();
- for (int i = 0; i < stalePlanOutputSlotList.size(); i++) {
- staleToNew.put(stalePlanOutputSlotList.get(i), childOutputSlotList.get(i));
- }
- List<Expression> groupByExpressionList = agg.getGroupByExprList();
- for (int i = 0; i < groupByExpressionList.size(); i++) {
- replaceSlot(staleToNew, groupByExpressionList, groupByExpressionList.get(i), i);
- }
- List<NamedExpression> mergeOutputExpressionList = agg.getOutputExpressionList();
- for (int i = 0; i < mergeOutputExpressionList.size(); i++) {
- replaceSlot(staleToNew, mergeOutputExpressionList, mergeOutputExpressionList.get(i), i);
- }
- LogicalAggregate mergeAgg = new LogicalAggregate(
- groupByExpressionList,
- mergeOutputExpressionList,
+ LogicalAggregate globalAggregate = new LogicalAggregate(
+ globalGroupByExprs,
+ globalOutputExprs,
true,
- AggPhase.FIRST_MERGE
+ AggPhase.GLOBAL
);
- return plan(mergeAgg, childPlan);
+ return plan(globalAggregate, plan(localAggregate, plan.child(0)));
}).toRule(RuleType.AGGREGATE_DISASSEMBLE);
}
- private org.apache.doris.catalog.AggregateFunction findAggFunc(AggregateFunction functionCall) {
- FunctionName functionName = new FunctionName(functionCall.getName());
- List<Expression> expressionList = functionCall.getArguments();
- List<Type> staleTypeList = expressionList.stream().map(Expression::getDataType)
- .map(DataType::toCatalogDataType).collect(Collectors.toList());
- Function staleFuncDesc = new Function(functionName, staleTypeList,
- functionCall.getDataType().toCatalogDataType(),
- // I think an aggregate function will never have a variable length parameters
- false);
- Function staleFunc = Catalog.getCurrentCatalog()
- .getFunction(staleFuncDesc, CompareMode.IS_IDENTICAL);
- Preconditions.checkArgument(staleFunc instanceof org.apache.doris.catalog.AggregateFunction);
- return (org.apache.doris.catalog.AggregateFunction) staleFunc;
- }
+ @SuppressWarnings("InnerClassMayBeStatic")
+ private static class ExpressionReplacer
+ extends ExpressionVisitor<Expression, Map<Expression, Expression>> {
+ private static final ExpressionReplacer INSTANCE = new ExpressionReplacer();
- @SuppressWarnings("unchecked")
- private <T extends Expression> void replaceSlot(Map<Slot, Slot> staleToNew,
- List<T> expressionList, Expression root, int index) {
- if (index != -1) {
- if (root instanceof Slot) {
- Slot v = staleToNew.get(root);
- if (v == null) {
- return;
+ @Override
+ public Expression visit(Expression expr, Map<Expression, Expression> substitutionMap) {
+ // TODO: we need to do sub tree match and replace. but we do not have semanticEquals now.
+ // e.g. a + 1 + 2 in output expression should be replaced by
+ // (slot reference to update phase out (a + 1)) + 2, if we do group by a + 1
+ // currently, we could only handle output expression same with group by expression
+ if (substitutionMap.containsKey(expr)) {
+ return substitutionMap.get(expr);
+ } else {
+ List<Expression> newChildren = new ArrayList<>();
Review Comment:
I think you can extends DefaultExpressionRewriter, and replace this line to `return super.visit(expr, substitutionMap)`
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggregateDisassemble.java:
##########
@@ -17,144 +17,156 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.analysis.FunctionName;
-import org.apache.doris.catalog.Catalog;
-import org.apache.doris.catalog.Function;
-import org.apache.doris.catalog.Function.CompareMode;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.operators.Operator;
import org.apache.doris.nereids.operators.plans.AggPhase;
import org.apache.doris.nereids.operators.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnaryPlan;
-import com.clearspring.analytics.util.Lists;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
- * TODO: if instance count is 1, shouldn't disassemble the agg operator
* Used to generate the merge agg node for distributed execution.
- * Do this in following steps:
- * 1. clone output expr list, find all agg function
- * 2. set found agg function intermediaType
- * 3. create new child plan rooted at new local agg
- * 4. update the slot referenced by expr of merge agg
- * 5. create plan rooted at merge agg, return it.
+ * NOTICE: GLOBAL output expressions' ExprId should SAME with ORIGIN output expressions' ExprId.
+ * If we have a query: SELECT SUM(v1 * v2) + 1 FROM t GROUP BY k + 1
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(k + 1) #1, Alias(SUM(v1 * v2) + 1) #2], groupByExpr: [k + 1])
+ * +-- childPlan
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [Alias(b) #1, Alias(SUM(a) + 1) #2], groupByExpr: [b])
+ * +-- Aggregate(phase: [LOCAL], outputExpr: [SUM(v1 * v2) as a, (k + 1) as b], groupByExpr: [k + 1])
+ * +-- childPlan
+ *
+ * TODO:
+ * 1. use different class represent different phase aggregate
+ * 2. if instance count is 1, shouldn't disassemble the agg operator
+ * 3. we need another rule to removing duplicated expressions in group by expression list
*/
public class AggregateDisassemble extends OneRewriteRuleFactory {
@Override
public Rule<Plan> build() {
return logicalAggregate().when(p -> {
- LogicalAggregate logicalAggregation = p.getOperator();
- return !logicalAggregation.isDisassembled();
+ LogicalAggregate logicalAggregate = p.getOperator();
+ return !logicalAggregate.isDisassembled();
}).thenApply(ctx -> {
- Plan plan = ctx.root;
- Operator operator = plan.getOperator();
- LogicalAggregate agg = (LogicalAggregate) operator;
- List<NamedExpression> outputExpressionList = agg.getOutputExpressionList();
- List<NamedExpression> intermediateAggExpressionList = Lists.newArrayList();
- // TODO: shouldn't extract agg function from this field.
- for (NamedExpression namedExpression : outputExpressionList) {
- namedExpression = (NamedExpression) namedExpression.clone();
- List<AggregateFunction> functionCallList =
- namedExpression.collect(org.apache.doris.catalog.AggregateFunction.class::isInstance);
- // TODO: we will have another mechanism to get corresponding stale agg func.
- for (AggregateFunction functionCall : functionCallList) {
- org.apache.doris.catalog.AggregateFunction staleAggFunc = findAggFunc(functionCall);
- Type staleIntermediateType = staleAggFunc.getIntermediateType();
- Type staleRetType = staleAggFunc.getReturnType();
- if (staleIntermediateType != null && !staleIntermediateType.equals(staleRetType)) {
- functionCall.setIntermediate(DataType.convertFromCatalogDataType(staleIntermediateType));
+ LogicalUnaryPlan<LogicalAggregate, GroupPlan> plan = ctx.root;
+ LogicalAggregate aggregate = plan.getOperator();
+ List<NamedExpression> originOutputExprs = aggregate.getOutputExpressionList();
+ List<Expression> originGroupByExprs = aggregate.getGroupByExpressionList();
+
+ // 1. generate a map from local aggregate output to global aggregate expr substitution.
+ // inputSubstitutionMap use for replacing expression in global aggregate
+ // replace rule is:
+ // a: Expression is a group by key and is a slot reference. e.g. group by k1
+ // b. Expression is a group by key and is an expression. e.g. group by k1 + 1
+ // c. Expression is an aggregate function. e.g. sum(v1) in select list
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | situation | origin expression | local output expression | expression in global aggregate |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | a | Ref(k1)#1 | Ref(k1)#1 | Ref(k1)#1 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | b | Ref(k1)#1 + 1 | A(Ref(k1)#1 + 1, key)#2 | Ref(key)#2 |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // | c | A(AF(v1#1), 'af')#2 | A(AF(v1#1), 'af')#3 | AF(af#3) |
+ // +-----------+---------------------+-------------------------+--------------------------------+
+ // NOTICE: Ref: SlotReference, A: Alias, AF: AggregateFunction, #x: ExprId x
+ // 2. collect local aggregate output expressions and local aggregate group by expression list
Review Comment:
great comment
##########
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateDisassembleTest.java:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.nereids.PlannerContext;
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
+import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.operators.plans.AggPhase;
+import org.apache.doris.nereids.operators.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.operators.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble;
+import org.apache.doris.nereids.trees.expressions.Add;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Literal;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Sum;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.Plans;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnaryPlan;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+import java.util.List;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class AggregateDisassembleTest implements Plans {
+ private Plan rStudent;
+
+ @BeforeAll
+ public final void beforeAll() {
+ Table student = new Table(0L, "student", Table.TableType.OLAP,
+ ImmutableList.of(new Column("id", Type.INT, true, AggregateType.NONE, true, "0", ""),
+ new Column("name", Type.STRING, true, AggregateType.NONE, true, "", ""),
+ new Column("age", Type.INT, true, AggregateType.NONE, true, "", "")));
+ rStudent = plan(new LogicalOlapScan(student, ImmutableList.of("student")));
+ }
+
+ /**
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [age, SUM(id) as sum], groupByExpr: [age])
+ * +--childPlan(id, name, age)
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [a, SUM(b) as c], groupByExpr: [a])
+ * +--Aggregate(phase: [LOCAL], outputExpr: [age as a, SUM(id) as b], groupByExpr: [age])
+ * +--childPlan(id, name, age)
+ */
+ @Test
+ public void slotReferenceGroupBy() {
+ List<Expression> groupExpressionList = Lists.newArrayList(
+ rStudent.getOutput().get(2).toSlot());
+ List<NamedExpression> outputExpressionList = Lists.newArrayList(
+ rStudent.getOutput().get(2).toSlot(),
+ new Alias<>(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
+ Plan root = plan(new LogicalAggregate(groupExpressionList, outputExpressionList), rStudent);
+
+ Memo memo = new Memo();
+ memo.initialize(root);
+
+ PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
+ JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
+ RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
+ ImmutableList.of(new AggregateDisassemble().build()), jobContext);
+ plannerContext.pushJob(rewriteTopDownJob);
+ plannerContext.getJobScheduler().executeJobPool(plannerContext);
+
+ Plan after = memo.copyOut();
+
+ Assertions.assertTrue(after instanceof LogicalUnaryPlan);
+ Assertions.assertTrue(after.getOperator() instanceof LogicalAggregate);
+ Assertions.assertTrue(after.child(0) instanceof LogicalUnaryPlan);
+ LogicalAggregate global = (LogicalAggregate) after.getOperator();
+ LogicalAggregate local = (LogicalAggregate) after.child(0).getOperator();
+ Assertions.assertEquals(AggPhase.GLOBAL, global.getAggPhase());
+ Assertions.assertEquals(AggPhase.LOCAL, local.getAggPhase());
+
+ Expression localOutput0 = rStudent.getOutput().get(2).toSlot();
+ Expression localOutput1 = new Sum(rStudent.getOutput().get(0).toSlot());
+ Expression localGroupBy = rStudent.getOutput().get(2).toSlot();
+
+ Assertions.assertEquals(2, local.getOutputExpressionList().size());
+ Assertions.assertTrue(local.getOutputExpressionList().get(0) instanceof SlotReference);
+ Assertions.assertEquals(localOutput0, local.getOutputExpressionList().get(0));
+ Assertions.assertTrue(local.getOutputExpressionList().get(1) instanceof Alias);
+ Assertions.assertEquals(localOutput1, local.getOutputExpressionList().get(1).child(0));
+ Assertions.assertEquals(1, local.getGroupByExpressionList().size());
+ Assertions.assertEquals(localGroupBy, local.getGroupByExpressionList().get(0));
+
+ Expression globalOutput0 = local.getOutputExpressionList().get(0).toSlot();
+ Expression globalOutput1 = new Sum(local.getOutputExpressionList().get(1).toSlot());
+ Expression globalGroupBy = local.getOutputExpressionList().get(0).toSlot();
+
+ Assertions.assertEquals(2, global.getOutputExpressionList().size());
+ Assertions.assertTrue(global.getOutputExpressionList().get(0) instanceof SlotReference);
+ Assertions.assertEquals(globalOutput0, global.getOutputExpressionList().get(0));
+ Assertions.assertTrue(global.getOutputExpressionList().get(1) instanceof Alias);
+ Assertions.assertEquals(globalOutput1, global.getOutputExpressionList().get(1).child(0));
+ Assertions.assertEquals(1, global.getGroupByExpressionList().size());
+ Assertions.assertEquals(globalGroupBy, global.getGroupByExpressionList().get(0));
+
+ // check id:
+ Assertions.assertEquals(outputExpressionList.get(0).getExprId(),
+ global.getOutputExpressionList().get(0).getExprId());
+ Assertions.assertEquals(outputExpressionList.get(1).getExprId(),
+ global.getOutputExpressionList().get(1).getExprId());
+ }
+
+ /**
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [(age + 1) as key, SUM(id) as sum], groupByExpr: [age + 1])
+ * +--childPlan(id, name, age)
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [a, SUM(b) as c], groupByExpr: [a])
+ * +--Aggregate(phase: [LOCAL], outputExpr: [(age + 1) as a, SUM(id) as b], groupByExpr: [age + 1])
+ * +--childPlan(id, name, age)
+ */
+ @Test
+ public void aliasGroupBy() {
+ List<Expression> groupExpressionList = Lists.newArrayList(
+ new Add<>(rStudent.getOutput().get(2).toSlot(), new Literal(1)));
+ List<NamedExpression> outputExpressionList = Lists.newArrayList(
+ new Alias<>(new Add<>(rStudent.getOutput().get(2).toSlot(), new Literal(1)), "key"),
+ new Alias<>(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
+ Plan root = plan(new LogicalAggregate(groupExpressionList, outputExpressionList), rStudent);
+
+ Memo memo = new Memo();
+ memo.initialize(root);
+
+ PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
+ JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
+ RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
+ ImmutableList.of(new AggregateDisassemble().build()), jobContext);
+ plannerContext.pushJob(rewriteTopDownJob);
+ plannerContext.getJobScheduler().executeJobPool(plannerContext);
+
+ Plan after = memo.copyOut();
+
+ Assertions.assertTrue(after instanceof LogicalUnaryPlan);
+ Assertions.assertTrue(after.getOperator() instanceof LogicalAggregate);
+ Assertions.assertTrue(after.child(0) instanceof LogicalUnaryPlan);
+ LogicalAggregate global = (LogicalAggregate) after.getOperator();
+ LogicalAggregate local = (LogicalAggregate) after.child(0).getOperator();
+ Assertions.assertEquals(AggPhase.GLOBAL, global.getAggPhase());
+ Assertions.assertEquals(AggPhase.LOCAL, local.getAggPhase());
+
+ Expression localOutput0 = new Add<>(rStudent.getOutput().get(2).toSlot(), new Literal(1));
+ Expression localOutput1 = new Sum(rStudent.getOutput().get(0).toSlot());
+ Expression localGroupBy = new Add<>(rStudent.getOutput().get(2).toSlot(), new Literal(1));
+
+ Assertions.assertEquals(2, local.getOutputExpressionList().size());
+ Assertions.assertTrue(local.getOutputExpressionList().get(0) instanceof Alias);
+ Assertions.assertEquals(localOutput0, local.getOutputExpressionList().get(0).child(0));
+ Assertions.assertTrue(local.getOutputExpressionList().get(1) instanceof Alias);
+ Assertions.assertEquals(localOutput1, local.getOutputExpressionList().get(1).child(0));
+ Assertions.assertEquals(1, local.getGroupByExpressionList().size());
+ Assertions.assertEquals(localGroupBy, local.getGroupByExpressionList().get(0));
+
+ Expression globalOutput0 = local.getOutputExpressionList().get(0).toSlot();
+ Expression globalOutput1 = new Sum(local.getOutputExpressionList().get(1).toSlot());
+ Expression globalGroupBy = local.getOutputExpressionList().get(0).toSlot();
+
+ Assertions.assertEquals(2, global.getOutputExpressionList().size());
+ Assertions.assertTrue(global.getOutputExpressionList().get(0) instanceof Alias);
+ Assertions.assertEquals(globalOutput0, global.getOutputExpressionList().get(0).child(0));
+ Assertions.assertTrue(global.getOutputExpressionList().get(1) instanceof Alias);
+ Assertions.assertEquals(globalOutput1, global.getOutputExpressionList().get(1).child(0));
+ Assertions.assertEquals(1, global.getGroupByExpressionList().size());
+ Assertions.assertEquals(globalGroupBy, global.getGroupByExpressionList().get(0));
+
+ // check id:
+ Assertions.assertEquals(outputExpressionList.get(0).getExprId(),
+ global.getOutputExpressionList().get(0).getExprId());
+ Assertions.assertEquals(outputExpressionList.get(1).getExprId(),
+ global.getOutputExpressionList().get(1).getExprId());
+ }
+
+ /**
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [SUM(id) as sum], groupByExpr: [])
+ * +--childPlan(id, name, age)
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [SUM(b) as b], groupByExpr: [])
+ * +--Aggregate(phase: [LOCAL], outputExpr: [SUM(id) as a], groupByExpr: [])
+ * +--childPlan(id, name, age)
+ */
+ @Test
+ public void globalAggregate() {
+ List<Expression> groupExpressionList = Lists.newArrayList();
+ List<NamedExpression> outputExpressionList = Lists.newArrayList(
+ new Alias<>(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
+ Plan root = plan(new LogicalAggregate(groupExpressionList, outputExpressionList), rStudent);
+
+ Memo memo = new Memo();
+ memo.initialize(root);
+
+ PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
+ JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
+ RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
+ ImmutableList.of(new AggregateDisassemble().build()), jobContext);
+ plannerContext.pushJob(rewriteTopDownJob);
+ plannerContext.getJobScheduler().executeJobPool(plannerContext);
+
+ Plan after = memo.copyOut();
+
+ Assertions.assertTrue(after instanceof LogicalUnaryPlan);
+ Assertions.assertTrue(after.getOperator() instanceof LogicalAggregate);
+ Assertions.assertTrue(after.child(0) instanceof LogicalUnaryPlan);
+ LogicalAggregate global = (LogicalAggregate) after.getOperator();
+ LogicalAggregate local = (LogicalAggregate) after.child(0).getOperator();
+ Assertions.assertEquals(AggPhase.GLOBAL, global.getAggPhase());
+ Assertions.assertEquals(AggPhase.LOCAL, local.getAggPhase());
+
+ Expression localOutput0 = new Sum(rStudent.getOutput().get(0).toSlot());
+
+ Assertions.assertEquals(1, local.getOutputExpressionList().size());
+ Assertions.assertTrue(local.getOutputExpressionList().get(0) instanceof Alias);
+ Assertions.assertEquals(localOutput0, local.getOutputExpressionList().get(0).child(0));
+ Assertions.assertEquals(0, local.getGroupByExpressionList().size());
+
+ Expression globalOutput0 = new Sum(local.getOutputExpressionList().get(0).toSlot());
+
+ Assertions.assertEquals(1, global.getOutputExpressionList().size());
+ Assertions.assertTrue(global.getOutputExpressionList().get(0) instanceof Alias);
+ Assertions.assertEquals(globalOutput0, global.getOutputExpressionList().get(0).child(0));
+ Assertions.assertEquals(0, global.getGroupByExpressionList().size());
+
+ // check id:
+ Assertions.assertEquals(outputExpressionList.get(0).getExprId(),
+ global.getOutputExpressionList().get(0).getExprId());
+ }
+
+ /**
+ * the initial plan is:
+ * Aggregate(phase: [GLOBAL], outputExpr: [SUM(id) as sum], groupByExpr: [age])
+ * +--childPlan(id, name, age)
+ * we should rewrite to:
+ * Aggregate(phase: [GLOBAL], outputExpr: [SUM(b) as c], groupByExpr: [a])
Review Comment:
alias `c` should keep same as origin alias `sum`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org