You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2022/07/08 11:22:55 UTC
[doris] branch master updated: [feature](nereides) support sort translator (#10678)
This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e6da00bb26 [feature](nereides) support sort translator (#10678)
e6da00bb26 is described below
commit e6da00bb261ef6a04207336e318e64b2558193c3
Author: zhengshiJ <32...@users.noreply.github.com>
AuthorDate: Fri Jul 8 19:22:48 2022 +0800
[feature](nereides) support sort translator (#10678)
Physical sort:
* 1. Build sortInfo
* There are two types of slotRef:
* one is generated by the previous node, collectively called old.
* the other is newly generated by the sort node, collectively called new.
* Filling of sortInfo related data structures,
* a. ordering use newSlotRef.
* b. sortTupleSlotExprs use oldSlotRef.
* 2. Create sortNode
* 3. Create mergeFragment
TODO:
1.Currently, columns that do not exist in select but exist in order by cannot be parsed.
eg: select key from table order by value;
2.For the combination of Literal and slotRefrance in select, there is a problem with parsing,
eg: select key ,(10-value) from table;
---
.../java/org/apache/doris/analysis/SortInfo.java | 4 ++
.../org/apache/doris/nereids/NereidsPlanner.java | 6 +--
.../glue/translator/PhysicalPlanTranslator.java | 56 +++++++++++++++++++---
.../glue/translator/PlanTranslatorContext.java | 13 ++++-
.../java/org/apache/doris/planner/PlanNode.java | 7 +++
.../java/org/apache/doris/planner/SortNode.java | 23 +++++++++
.../java/org/apache/doris/qe/StmtExecutor.java | 1 -
7 files changed, 98 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
index 05763e8d07..63394db4cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
@@ -141,6 +141,10 @@ public class SortInfo {
this.sortTupleSlotExprs = sortTupleSlotExprs;
}
+ public void setSortTupleDesc(TupleDescriptor tupleDesc) {
+ sortTupleDesc = tupleDesc;
+ }
+
public TupleDescriptor getSortTupleDescriptor() {
return sortTupleDesc;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index bc7aed0bf1..23fafa518d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -25,7 +25,6 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Id;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
@@ -39,7 +38,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -104,8 +102,8 @@ public class NereidsPlanner extends Planner {
outputCandidates.put(slotDescriptor.getId().asInt(), slotRef);
}
}
- physicalPlan.getOutput().stream().map(Slot::getExprId)
- .map(Id::asInt).forEach(i -> outputExprs.add(outputCandidates.get(i)));
+ physicalPlan.getOutput().stream()
+ .forEach(i -> outputExprs.add(planTranslatorContext.findExpr(i)));
root.setOutputExprs(outputExprs);
root.getPlanRoot().convertToVectoriezd();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 511ac61fb4..0dd6b9e859 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -204,6 +204,26 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
return planFragment;
}
+ /**
+ * Physical sort:
+ * 1. Build sortInfo
+ * There are two types of slotRef:
+ * one is generated by the previous node, collectively called old.
+ * the other is newly generated by the sort node, collectively called new.
+ * Filling of sortInfo related data structures,
+ * a. ordering use newSlotRef.
+ * b. sortTupleSlotExprs use oldSlotRef.
+ * 2. Create sortNode
+ * 3. Create mergeFragment
+ * TODO: When the slotRef of sort is currently generated,
+ * it will be based on the expression in select and orderBy expression in to ensure the uniqueness of slotRef.
+ * But eg:
+ * select a+1 from table order by a+1;
+ * the expressions of the two are inconsistent.
+ * The former will perform an additional Alisa.
+ * Currently we cannot test whether this will have any effect.
+ * After a+1 can be parsed , reprocessing.
+ */
@Override
public PlanFragment visitPhysicalSort(PhysicalUnaryPlan<PhysicalHeapSort, Plan> sort,
PlanTranslatorContext context) {
@@ -211,24 +231,35 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
PhysicalHeapSort physicalHeapSort = sort.getOperator();
long limit = physicalHeapSort.getLimit();
// TODO: need to discuss how to process field: SortNode::resolvedTupleExprs
- List<Expr> execOrderingExprList = Lists.newArrayList();
+ List<Expr> oldOrderingExprList = Lists.newArrayList();
List<Boolean> ascOrderList = Lists.newArrayList();
List<Boolean> nullsFirstParamList = Lists.newArrayList();
List<OrderKey> orderKeyList = physicalHeapSort.getOrderKeys();
+ // 1.Get previous slotRef
orderKeyList.forEach(k -> {
- execOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
+ oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
ascOrderList.add(k.isAsc());
nullsFirstParamList.add(k.isNullFirst());
});
-
+ List<Expr> sortTupleOutputList = new ArrayList<>();
List<Slot> outputList = sort.getOutput();
- TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, null);
- SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc);
-
+ outputList.forEach(k -> {
+ sortTupleOutputList.add(ExpressionTranslator.translate(k, context));
+ });
+ // 2. Generate new Tuple
+ TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, context, null);
+ // 3. Get current slotRef
+ List<Expr> newOrderingExprList = Lists.newArrayList();
+ orderKeyList.forEach(k -> {
+ newOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
+ });
+ // 4. fill in SortInfo members
+ SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc);
PlanNode childNode = childFragment.getPlanRoot();
// TODO: notice topN
SortNode sortNode = new SortNode(context.nextNodeId(), childNode, sortInfo, true,
physicalHeapSort.hasLimit(), physicalHeapSort.getOffset());
+ sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList);
childFragment.addPlanRoot(sortNode);
if (!childFragment.isPartitioned()) {
return childFragment;
@@ -362,6 +393,19 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
return tupleDescriptor;
}
+ private TupleDescriptor generateTupleDesc(List<Slot> slotList, List<OrderKey> orderKeyList,
+ PlanTranslatorContext context, Table table) {
+ TupleDescriptor tupleDescriptor = context.generateTupleDesc();
+ tupleDescriptor.setTable(table);
+ for (Slot slot : slotList) {
+ context.createSlotDesc(tupleDescriptor, (SlotReference) slot);
+ }
+ for (OrderKey orderKey : orderKeyList) {
+ context.createSlotDesc(tupleDescriptor, orderKey.getExpr());
+ }
+ return tupleDescriptor;
+ }
+
private PlanFragment createParentFragment(PlanFragment childFragment, DataPartition parentPartition,
PlanTranslatorContext ctx) {
ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), childFragment.getPlanRoot(), false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index afde344669..cc3337bbd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -106,7 +106,7 @@ public class PlanTranslatorContext {
* Create SlotDesc and add it to the mappings from expression to the stales epxr
*/
public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference slotReference) {
- SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc, slotReference.getExprId().asInt());
+ SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
Column column = slotReference.getColumn();
// Only the SlotDesc that in the tuple generated for scan node would have corresponding column.
if (column != null) {
@@ -118,6 +118,17 @@ public class PlanTranslatorContext {
return slotDescriptor;
}
+ /**
+ * Create slotDesc with Expression.
+ */
+ public void createSlotDesc(TupleDescriptor tupleDesc, Expression expression) {
+ if (!expressionToExecExpr.containsKey(expression)) {
+ SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
+ slotDescriptor.setType(expression.getDataType().toCatalogDataType());
+ this.addSlotRefMapping(expression, new SlotRef(slotDescriptor));
+ }
+ }
+
public TupleDescriptor getTupleDesc(TupleId tupleId) {
return descTable.getTupleDesc(tupleId);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index c2dc140d90..074a52888f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -999,4 +999,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
output.append("\n");
}
}
+
+ /**
+ * Supplement the information be needed for nodes generated by the new optimizer.
+ */
+ public void finalizeForNereids() {
+
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
index db08f4eb48..232e703555 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
@@ -22,11 +22,13 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.ExprId;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.SortInfo;
+import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
@@ -44,6 +46,7 @@ import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -264,4 +267,24 @@ public class SortNode extends PlanNode {
Expr.getIds(resolvedTupleExprs, null, result);
return new HashSet<>(result);
}
+
+ /**
+ * Supplement the information needed by be for the sort node.
+ */
+ public void finalizeForNereids(TupleDescriptor tupleDescriptor,
+ List<Expr> outputList, List<Expr> orderingExpr) {
+ List<Expr> sortTupleSlotExprs = new ArrayList<>();
+ sortTupleSlotExprs.addAll(outputList);
+ sortTupleSlotExprs.addAll(orderingExpr);
+ List<Expr> afterDeduplication = new ArrayList<>();
+ Set<ExprId> exprIds = new HashSet<>();
+ for (int i = 0; i < sortTupleSlotExprs.size(); i++) {
+ Expr expr = sortTupleSlotExprs.get(i);
+ if (!exprIds.contains(expr.getId())) {
+ afterDeduplication.add(expr);
+ }
+ }
+ info.setSortTupleDesc(tupleDescriptor);
+ info.setSortTupleSlotExprs(afterDeduplication);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 34321cee03..6478551678 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -978,7 +978,6 @@ public class StmtExecutor implements ProfileWriter {
return;
}
-
MysqlChannel channel = context.getMysqlChannel();
boolean isOutfileQuery = queryStmt.hasOutFileClause();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org