You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/20 14:07:00 UTC
[doris] 16/21: [stats](nereids) dump col stats for all physical plan node and cost details in memo #21902
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit aba06c06038890fc76c9166459501202ec398fe5
Author: minghong <en...@gmail.com>
AuthorDate: Wed Jul 19 14:10:26 2023 +0800
[stats](nereids) dump col stats for all physical plan node and cost details in memo #21902
1. print cost detail
2. dump col stats in memo
---
.../main/java/org/apache/doris/nereids/cost/CostV1.java | 2 +-
.../doris/nereids/jobs/cascades/CostAndEnforcerJob.java | 4 ++--
.../org/apache/doris/nereids/memo/GroupExpression.java | 12 ++++++++----
.../main/java/org/apache/doris/nereids/memo/Memo.java | 17 ++---------------
.../properties/EnforceMissingPropertiesHelper.java | 2 +-
.../java/org/apache/doris/statistics/Statistics.java | 11 +++++++++++
6 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
index 2747346390..b5c5b50bd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostV1.java
@@ -105,7 +105,7 @@ class CostV1 implements Cost {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append(cost).append("[").append((long) cpuCost).append("/")
+ sb.append("[").append((long) cpuCost).append("/")
.append((long) memoryCost).append("/").append((long) networkCost)
.append("/").append((long) penalty).append("]");
return sb.toString();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
index d8e79bd80f..37f5e5b681 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
@@ -141,7 +141,7 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
// Calculate cost
if (curChildIndex == 0 && prevChildIndex == -1) {
curNodeCost = CostCalculator.calculateCost(groupExpression, requestChildrenProperties);
- groupExpression.setCost(curNodeCost.getValue());
+ groupExpression.setCost(curNodeCost);
curTotalCost = curNodeCost;
}
@@ -256,7 +256,7 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
// recompute cost after adjusting property
curNodeCost = CostCalculator.calculateCost(groupExpression, requestChildrenProperties);
- groupExpression.setCost(curNodeCost.getValue());
+ groupExpression.setCost(curNodeCost);
curTotalCost = curNodeCost;
for (int i = 0; i < outputChildrenProperties.size(); i++) {
PhysicalProperties childProperties = outputChildrenProperties.get(i);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
index 6264dd2ce6..b0da8459f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
@@ -53,7 +53,7 @@ public class GroupExpression {
private static final EventProducer COST_STATE_TRACER = new EventProducer(CostStateUpdateEvent.class,
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(CostStateUpdateEvent.class,
EventChannel.LOG)));
- private double cost = 0.0;
+ private Cost cost;
private Group ownerGroup;
private final List<Group> children;
private final Plan plan;
@@ -284,11 +284,11 @@ public class GroupExpression {
this.ownerGroup = null;
}
- public double getCost() {
+ public Cost getCost() {
return cost;
}
- public void setCost(double cost) {
+ public void setCost(Cost cost) {
this.cost = cost;
}
@@ -328,7 +328,11 @@ public class GroupExpression {
} else {
builder.append("#").append(ownerGroup.getGroupId().asInt());
}
- builder.append(" cost=").append(format.format((long) cost));
+ if (cost != null) {
+ builder.append(" cost=").append(format.format((long) cost.getValue()) + " " + cost);
+ } else {
+ builder.append(" cost=null");
+ }
builder.append(" estRows=").append(format.format(estOutputRowCount));
builder.append(" children=[").append(Joiner.on(", ").join(
children.stream().map(Group::getGroupId).collect(Collectors.toList())))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 643ffe819c..72d673f0a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -34,14 +34,11 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.LeafPlan;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.statistics.Statistics;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -53,7 +50,6 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Optional;
import java.util.PriorityQueue;
import java.util.stream.Collectors;
@@ -736,17 +732,8 @@ public class Memo {
builder.append("root:").append(getRoot()).append("\n");
for (Group group : groups.values()) {
builder.append("\n\n").append(group);
- builder.append(" stats=").append(group.getStatistics()).append("\n");
- Statistics stats = group.getStatistics();
- if (stats != null && !group.getLogicalExpressions().isEmpty()) {
- Plan plan = group.getLogicalExpressions().get(0).getPlan();
- if (plan instanceof LogicalOlapScan || plan instanceof LogicalFileScan) {
- for (Entry e : stats.columnStatistics().entrySet()) {
- builder.append(" ").append(e.getKey()).append(":").append(e.getValue()).append("\n");
- }
- }
- }
-
+ builder.append(" stats").append("\n");
+ builder.append(group.getStatistics().detail(" "));
builder.append(" lowest Plan(cost, properties, plan, childrenRequires)");
group.getAllProperties().forEach(
prop -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
index bcbfffc0e7..2e9a8ac6bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
@@ -156,7 +156,7 @@ public class EnforceMissingPropertiesHelper {
oldOutputProperty, newOutputProperty));
enforcer.setEstOutputRowCount(enforcer.getOwnerGroup().getStatistics().getRowCount());
Cost enforcerCost = CostCalculator.calculateCost(enforcer, Lists.newArrayList(oldOutputProperty));
- enforcer.setCost(enforcerCost.getValue());
+ enforcer.setCost(enforcerCost);
curTotalCost = CostCalculator.addChildCost(enforcer.getPlan(),
enforcerCost,
curTotalCost,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
index d1872edddb..5c628aaba3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -243,4 +243,15 @@ public class Statistics {
}
}
}
+
+ public String detail(String prefix) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(prefix).append("rows=").append(rowCount).append("\n");
+ builder.append(prefix).append("tupleSize=").append(computeTupleSize()).append("\n");
+
+ for (Entry<Expression, ColumnStatistic> entry : expressionToColumnStats.entrySet()) {
+ builder.append(prefix).append(entry.getKey()).append(" -> ").append(entry.getValue()).append("\n");
+ }
+ return builder.toString();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org