You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/09 16:18:01 UTC
[doris] 09/13: [minor](Nereids) remove some invasive code of minidump in cascades framework (#20606)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cbfaec4c9c5da0cb7431d3fbf9be5083ddb92ee0
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Fri Jun 9 23:41:00 2023 +0800
[minor](Nereids) remove some invasive code of minidump in cascades framework (#20606)
---
.../java/org/apache/doris/nereids/cost/Cost.java | 14 +++++----
.../apache/doris/nereids/cost/CostCalculator.java | 16 ++++-------
.../org/apache/doris/nereids/cost/CostWeight.java | 9 ++----
.../nereids/jobs/cascades/CostAndEnforcerJob.java | 33 +++++++++-------------
.../jobs/joinorder/hypergraph/GraphSimplifier.java | 4 +--
.../apache/doris/nereids/memo/GroupExpression.java | 2 +-
.../org/apache/doris/nereids/memo/MemoTest.java | 2 +-
7 files changed, 35 insertions(+), 45 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java
index c0ec0b1956..18dfbad5e0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.cost;
+import org.apache.doris.qe.ConnectContext;
+
/**
* Cost encapsulate the real cost with double type.
* We do this because we want to customize the operation of adding child cost
@@ -28,8 +30,8 @@ public interface Cost {
/**
* This is for calculating the cost in simplifier
*/
- static Cost withRowCount(double rowCount, boolean enableNewCostModel) {
- if (enableNewCostModel) {
+ static Cost withRowCount(double rowCount) {
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
return new CostV2(0, rowCount, 0);
}
return new CostV1(rowCount);
@@ -38,15 +40,15 @@ public interface Cost {
/**
* return zero cost
*/
- static Cost zero(boolean enableNewCostModel) {
- if (enableNewCostModel) {
+ static Cost zero() {
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
return CostV2.zero();
}
return CostV1.zero();
}
- static Cost infinite(boolean enableNewCostModel) {
- if (enableNewCostModel) {
+ static Cost infinite() {
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
return CostV2.infinite();
}
return CostV1.infinite();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
index b1e1570171..7d6125494f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
@@ -23,6 +23,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpecReplicated;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.qe.ConnectContext;
import java.util.List;
@@ -32,11 +33,6 @@ import java.util.List;
@Developing
//TODO: memory cost and network cost should be estimated by byte size.
public class CostCalculator {
- private static boolean enableNewCostModel = false;
-
- public static void setEnableNewCostModel(boolean enableNewCostModel) {
- CostCalculator.enableNewCostModel = enableNewCostModel;
- }
/**
* Calculate cost for groupExpression
@@ -47,7 +43,7 @@ public class CostCalculator {
&& childrenProperties.get(1).getDistributionSpec() instanceof DistributionSpecReplicated) {
planContext.setBroadcastJoin();
}
- if (enableNewCostModel) {
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
CostModelV2 costModelV2 = new CostModelV2();
return groupExpression.getPlan().accept(costModelV2, planContext);
} else {
@@ -60,7 +56,7 @@ public class CostCalculator {
* Calculate cost without groupExpression
*/
public static Cost calculateCost(Plan plan, PlanContext planContext) {
- if (enableNewCostModel) {
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
CostModelV2 costModel = new CostModelV2();
return plan.accept(costModel, planContext);
} else {
@@ -70,9 +66,9 @@ public class CostCalculator {
}
public static Cost addChildCost(Plan plan, Cost planCost, Cost childCost, int index) {
- if (!enableNewCostModel) {
- return CostModelV1.addChildCost(plan, planCost, childCost, index);
+ if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) {
+ return CostModelV2.addChildCost(plan, planCost, childCost, index);
}
- return CostModelV2.addChildCost(plan, planCost, childCost, index);
+ return CostModelV1.addChildCost(plan, planCost, childCost, index);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java
index 5cd576ac28..a0bb8c7a87 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.cost;
+import org.apache.doris.qe.ConnectContext;
+
import com.google.common.base.Preconditions;
/**
@@ -37,7 +39,6 @@ public class CostWeight {
static final double NETWORK_WEIGHT = 1.5;
static final double DELAY = 0.5;
- static double nereidsCboPenaltyFactor = 0.7;
final double cpuWeight;
final double memoryWeight;
final double networkWeight;
@@ -69,7 +70,7 @@ public class CostWeight {
public static CostWeight get() {
return new CostWeight(CPU_WEIGHT, MEMORY_WEIGHT, NETWORK_WEIGHT,
- nereidsCboPenaltyFactor);
+ ConnectContext.get().getSessionVariable().getNereidsCboPenaltyFactor());
}
//TODO: add it in session variable
@@ -80,8 +81,4 @@ public class CostWeight {
public double weightSum(double cpuCost, double ioCost, double netCost) {
return cpuCost * cpuWeight + ioCost * ioWeight + netCost * networkWeight;
}
-
- public static void setNereidsCboPenaltyFactor(double nereidsCboPenaltyFactor) {
- CostWeight.nereidsCboPenaltyFactor = nereidsCboPenaltyFactor;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
index ff82addb25..b673bd6e87 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
@@ -20,7 +20,6 @@ package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.cost.Cost;
import org.apache.doris.nereids.cost.CostCalculator;
-import org.apache.doris.nereids.cost.CostWeight;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
@@ -47,6 +46,7 @@ import java.util.Optional;
* Inspired by NoisePage and ORCA-Paper.
*/
public class CostAndEnforcerJob extends Job implements Cloneable {
+
private static final Logger LOG = LogManager.getLogger(CostAndEnforcerJob.class);
// GroupExpression to optimize
@@ -110,10 +110,6 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
*/
@Override
public void execute() {
- CostCalculator.setEnableNewCostModel(
- context.getCascadesContext().getConnectContext().getSessionVariable().getEnableNewCostModel());
- CostWeight.setNereidsCboPenaltyFactor(
- context.getCascadesContext().getConnectContext().getSessionVariable().getNereidsCboPenaltyFactor());
if (groupExpression.isUnused()) {
return;
}
@@ -121,10 +117,8 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
countJobExecutionTimesOfGroupExpressions(groupExpression);
// Do init logic of root plan/groupExpr of `subplan`, only run once per task.
if (curChildIndex == -1) {
- curNodeCost = Cost.zero(
- context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
- curTotalCost = Cost.zero(
- context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
+ curNodeCost = Cost.zero();
+ curTotalCost = Cost.zero();
curChildIndex = 0;
// List<request property to children>
// [ child item: [leftProperties, rightProperties]]
@@ -196,8 +190,7 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
lowestCostExpr.getCostValueByProperties(requestChildProperty),
curChildIndex);
if (curTotalCost.getValue() > context.getCostUpperBound()) {
- curTotalCost = Cost.infinite(
- context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
+ curTotalCost = Cost.infinite();
}
// the request child properties will be covered by the output properties
// that corresponding to the request properties. so if we run a costAndEnforceJob of the same
@@ -248,14 +241,18 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
// if we come here, mean that we have some error in stats calculator and should fix it.
return false;
}
+
StatsCalculator statsCalculator = StatsCalculator.estimate(groupExpression,
context.getCascadesContext().getConnectContext().getSessionVariable().getForbidUnknownColStats(),
context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap(),
context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
- context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap()
- .putAll(statsCalculator.getTotalColumnStatisticMap());
- context.getCascadesContext().getConnectContext().getTotalHistogramMap()
- .putAll(statsCalculator.getTotalHistogramMap());
+ if (!context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()
+ && context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMinidump()) {
+ context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap()
+ .putAll(statsCalculator.getTotalColumnStatisticMap());
+ context.getCascadesContext().getConnectContext().getTotalHistogramMap()
+ .putAll(statsCalculator.getTotalHistogramMap());
+ }
// recompute cost after adjusting property
curNodeCost = CostCalculator.calculateCost(groupExpression, requestChildrenProperties);
@@ -328,10 +325,8 @@ public class CostAndEnforcerJob extends Job implements Cloneable {
lowestCostChildren.clear();
prevChildIndex = -1;
curChildIndex = 0;
- curTotalCost = Cost.zero(
- context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
- curNodeCost = Cost.zero(
- context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump());
+ curTotalCost = Cost.zero();
+ curNodeCost = Cost.zero();
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java
index adee7f662e..efe9896dd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java
@@ -82,7 +82,7 @@ public class GraphSimplifier {
}
for (Node node : graph.getNodes()) {
cacheStats.put(node.getNodeMap(), node.getGroup().getStatistics());
- cacheCost.put(node.getNodeMap(), Cost.withRowCount(node.getRowCount(), false));
+ cacheCost.put(node.getNodeMap(), Cost.withRowCount(node.getRowCount()));
}
circleDetector = new CircleDetector(edgeSize);
@@ -427,7 +427,7 @@ public class GraphSimplifier {
Statistics leftStats, Statistics rightStats) {
LogicalJoin join = edge.getJoin();
PlanContext planContext = new PlanContext(stats, leftStats, rightStats);
- Cost cost = Cost.zero(false);
+ Cost cost;
if (JoinUtils.shouldNestedLoopJoin(join)) {
PhysicalNestedLoopJoin nestedLoopJoin = new PhysicalNestedLoopJoin<>(
join.getJoinType(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
index 43f54ca494..b921c80000 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
@@ -77,7 +77,7 @@ public class GroupExpression {
// After mergeGroup(), source Group was cleaned up, but it may be in the Job Stack. So use this to mark and skip it.
private boolean isUnused = false;
- private ObjectId id = StatementScopeIdGenerator.newObjectId();
+ private final ObjectId id = StatementScopeIdGenerator.newObjectId();
public GroupExpression(Plan plan) {
this(plan, Lists.newArrayList());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
index 776b68fb10..43b1c48761 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
@@ -95,7 +95,7 @@ class MemoTest implements MemoPatternMatchSupported {
FakePlan fakePlan = new FakePlan();
GroupExpression srcParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(srcGroup));
Group srcParentGroup = new Group(new GroupId(0), srcParentExpression, new LogicalProperties(ArrayList::new));
- srcParentGroup.setBestPlan(srcParentExpression, Cost.zero(false), PhysicalProperties.ANY);
+ srcParentGroup.setBestPlan(srcParentExpression, Cost.zero(), PhysicalProperties.ANY);
GroupExpression dstParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(dstGroup));
Group dstParentGroup = new Group(new GroupId(1), dstParentExpression, new LogicalProperties(ArrayList::new));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org