You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/12/22 08:44:47 UTC
[iotdb] branch memoryDistribution1.0 created (now ad0de56f15)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch memoryDistribution1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at ad0de56f15 change long to int
This branch includes the following new commits:
new 370fe9650d Fix memory distribution bug
new ad0de56f15 change long to int
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/02: Fix memory distribution bug
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch memoryDistribution1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 370fe9650d521a7f8a5bce74bff76e7b3b01394d
Author: Alima777 <wx...@gmail.com>
AuthorDate: Thu Dec 22 16:39:00 2022 +0800
Fix memory distribution bug
---
.../plan/planner/MemoryDistributionCalculator.java | 48 +++++++---------------
1 file changed, 14 insertions(+), 34 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
index d7f0e7f223..8d420ee53c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
@@ -77,18 +77,10 @@ import static org.apache.iotdb.db.mpp.plan.constant.DataNodeEndPoints.isSameNode
public class MemoryDistributionCalculator
extends PlanVisitor<Void, MemoryDistributionCalculator.MemoryDistributionContext> {
/** This map is used to calculate the total split of memory */
- private final Map<PlanNodeId, List<PlanNodeId>> exchangeMap;
-
- public MemoryDistributionCalculator() {
- this.exchangeMap = new HashMap<>();
- }
+ private int exchangeNum;
public long calculateTotalSplit() {
- long res = 0;
- for (List<PlanNodeId> l : exchangeMap.values()) {
- res += l.size();
- }
- return res;
+ return exchangeNum;
}
@Override
@@ -124,29 +116,17 @@ public class MemoryDistributionCalculator
});
}
+ /**
+ * We do not distinguish LocalSourceHandle/SourceHandle by not letting LocalSinkHandle update
+ */
@Override
public Void visitExchange(ExchangeNode node, MemoryDistributionContext context) {
- // we do not distinguish LocalSourceHandle/SourceHandle by not letting LocalSinkHandle update
- // the map
- if (context == null) {
- // context == null means this ExchangeNode has no father
- exchangeMap
- .computeIfAbsent(node.getPlanNodeId(), x -> new ArrayList<>())
- .add(node.getPlanNodeId());
- } else {
- if (context.memoryDistributionType.equals(
- MemoryDistributionType.CONSUME_ALL_CHILDREN_AT_THE_SAME_TIME)) {
- exchangeMap
- .computeIfAbsent(context.planNodeId, x -> new ArrayList<>())
- .add(node.getPlanNodeId());
- } else if (context.memoryDistributionType.equals(
- MemoryDistributionType.CONSUME_CHILDREN_ONE_BY_ONE)
- && !exchangeMap.containsKey(context.planNodeId)) {
- // All children share one split, thus only one node needs to be put into the map
- exchangeMap
- .computeIfAbsent(context.planNodeId, x -> new ArrayList<>())
- .add(node.getPlanNodeId());
- }
+ // context == null means this ExchangeNode doesn't have a father
+ if (context == null || context.memoryDistributionType.equals(MemoryDistributionType.CONSUME_ALL_CHILDREN_AT_THE_SAME_TIME)) {
+ exchangeNum++;
+ } else if (!context.exchangeAdded){
+ context.exchangeAdded = true;
+ exchangeNum++;
}
return null;
}
@@ -156,10 +136,9 @@ public class MemoryDistributionCalculator
// LocalSinkHandle and LocalSourceHandle are one-to-one mapped and only LocalSourceHandle do the
// update
if (!isSameNode(node.getDownStreamEndpoint())) {
- exchangeMap
- .computeIfAbsent(node.getDownStreamPlanNodeId(), x -> new ArrayList<>())
- .add(node.getDownStreamPlanNodeId());
+ exchangeNum++;
}
+ node.getChild().accept(this, context);
return null;
}
@@ -458,6 +437,7 @@ public class MemoryDistributionCalculator
static class MemoryDistributionContext {
final PlanNodeId planNodeId;
+ boolean exchangeAdded = false;
final MemoryDistributionType memoryDistributionType;
MemoryDistributionContext(
[iotdb] 02/02: change long to int
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch memoryDistribution1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ad0de56f153de771a874609b3322178f0448074a
Author: Alima777 <wx...@gmail.com>
AuthorDate: Thu Dec 22 16:40:10 2022 +0800
change long to int
---
.../org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java | 2 +-
.../apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
index 5b6d2abc86..36d53f6279 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
@@ -126,7 +126,7 @@ public class LocalExecutionPlanner {
private void setMemoryLimitForHandle(TFragmentInstanceId fragmentInstanceId, PlanNode plan) {
MemoryDistributionCalculator visitor = new MemoryDistributionCalculator();
plan.accept(visitor, null);
- long totalSplit = visitor.calculateTotalSplit();
+ int totalSplit = visitor.calculateTotalSplit();
if (totalSplit == 0) {
return;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
index 8d420ee53c..d18c7c36d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/MemoryDistributionCalculator.java
@@ -79,7 +79,7 @@ public class MemoryDistributionCalculator
/** This map is used to calculate the total split of memory */
private int exchangeNum;
- public long calculateTotalSplit() {
+ public int calculateTotalSplit() {
return exchangeNum;
}