You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/26 12:36:01 UTC
[doris] 08/09: [fix](nereids) set proper sort info to scan node to enable TopN-opt (#21148)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8f79e264f11c4c755976467c4ee2ebfdabc7323c
Author: minghong <en...@gmail.com>
AuthorDate: Mon Jun 26 19:54:37 2023 +0800
[fix](nereids) set proper sort info to scan node to enable TopN-opt (#21148)
---
.../java/org/apache/doris/analysis/SortInfo.java | 7 ++
.../glue/translator/PhysicalPlanTranslator.java | 75 +++++++++++++++++++++-
2 files changed, 81 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
index 0dc41c037d..a4d5962bc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
@@ -139,6 +139,13 @@ public class SortInfo {
return materializedOrderingExprs;
}
+ public void addMaterializedOrderingExpr(Expr expr) {
+ if (materializedOrderingExprs == null) {
+ materializedOrderingExprs = Lists.newArrayList();
+ }
+ materializedOrderingExprs.add(expr);
+ }
+
public List<Expr> getSortTupleSlotExprs() {
return sortTupleSlotExprs;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 10cb05acba..fa0bb2a303 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -160,6 +160,7 @@ import org.apache.doris.planner.external.HiveScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TFetchOption;
@@ -1069,7 +1070,25 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanNode child = sortNode.getChild(0);
Preconditions.checkArgument(child instanceof OlapScanNode,
"topN opt expect OlapScanNode, but we get " + child);
- ((OlapScanNode) child).setUseTopnOpt(true);
+ OlapScanNode scanNode = ((OlapScanNode) child);
+ scanNode.setUseTopnOpt(true);
+ }
+ // push sort to scan opt
+ if (sortNode.getChild(0) instanceof OlapScanNode) {
+ OlapScanNode scanNode = ((OlapScanNode) sortNode.getChild(0));
+ if (checkPushSort(sortNode, scanNode.getOlapTable())) {
+ SortInfo sortInfo = sortNode.getSortInfo();
+ scanNode.setSortInfo(sortInfo);
+ scanNode.getSortInfo().setSortTupleSlotExprs(sortNode.getResolvedTupleExprs());
+ for (Expr expr : sortInfo.getOrderingExprs()) {
+ scanNode.getSortInfo().addMaterializedOrderingExpr(expr);
+ }
+ if (sortNode.getOffset() > 0) {
+ scanNode.setSortLimit(sortNode.getLimit() + sortNode.getOffset());
+ } else {
+ scanNode.setSortLimit(sortNode.getLimit());
+ }
+ }
}
addPlanRoot(currentFragment, sortNode, topN);
} else {
@@ -1094,6 +1113,60 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return currentFragment;
}
+ /**
+ * topN opt: using storage data ordering to accelerate topn operation.
+ * refer pr: optimize topn query if order by columns is prefix of sort keys of table (#10694)
+ */
+ public boolean checkPushSort(SortNode sortNode, OlapTable olapTable) {
+ // Ensure limit is less then threshold
+ if (sortNode.getLimit() <= 0
+ || sortNode.getLimit() > ConnectContext.get().getSessionVariable().topnOptLimitThreshold) {
+ return false;
+ }
+
+ // Ensure all isAscOrder is same, ande length != 0.
+ // Can't be zorder.
+ if (sortNode.getSortInfo().getIsAscOrder().stream().distinct().count() != 1
+ || olapTable.isZOrderSort()) {
+ return false;
+ }
+
+ // Tablet's order by key only can be the front part of schema.
+ // Like: schema: a.b.c.d.e.f.g order by key: a.b.c (no a,b,d)
+ // Do **prefix match** to check if order by key can be pushed down.
+ // olap order by key: a.b.c.d
+ // sort key: (a) (a,b) (a,b,c) (a,b,c,d) is ok
+ // (a,c) (a,c,d), (a,c,b) (a,c,f) (a,b,c,d,e)is NOT ok
+ List<Expr> sortExprs = sortNode.getSortInfo().getOrderingExprs();
+ List<Boolean> nullsFirsts = sortNode.getSortInfo().getNullsFirst();
+ List<Boolean> isAscOrders = sortNode.getSortInfo().getIsAscOrder();
+ if (sortExprs.size() > olapTable.getDataSortInfo().getColNum()) {
+ return false;
+ }
+ for (int i = 0; i < sortExprs.size(); i++) {
+ // table key.
+ Column tableKey = olapTable.getFullSchema().get(i);
+ // sort slot.
+ Expr sortExpr = sortExprs.get(i);
+ if (sortExpr instanceof SlotRef) {
+ SlotRef slotRef = (SlotRef) sortExpr;
+ if (tableKey.equals(slotRef.getColumn())) {
+ // ORDER BY DESC NULLS FIRST can not be optimized to only read file tail,
+ // since NULLS is at file head but data is at tail
+ if (tableKey.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
private SortNode translateSortNode(AbstractPhysicalSort<? extends Plan> sort, PlanNode childNode,
PlanTranslatorContext context) {
List<Expr> oldOrderingExprList = Lists.newArrayList();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org