You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/14 09:33:24 UTC
(doris) branch master updated: [fix](pipelineX) fix multi be may be missing profiles #29914
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 67ef9e304e3 [fix](pipelineX) fix multi be may be missing profiles #29914
67ef9e304e3 is described below
commit 67ef9e304e3cf4e9ad4042f408b87e8d0112317d
Author: Mryange <59...@users.noreply.github.com>
AuthorDate: Sun Jan 14 17:33:17 2024 +0800
[fix](pipelineX) fix multi be may be missing profiles #29914
---
.../doris/common/profile/ExecutionProfile.java | 33 ++++++++++++++++++++--
.../main/java/org/apache/doris/qe/Coordinator.java | 3 ++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index 9584a6f1fa1..24bd2355c56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -33,11 +33,14 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* ExecutionProfile is used to collect profile of a complete query plan(including query or load).
@@ -72,6 +75,12 @@ public class ExecutionProfile {
// fragmentId -> dummy value
private MarkedCountDownLatch<Integer, Long> profileFragmentDoneSignal;
+ // fragmentId -> The number of BE without 'done.
+ private Map<Integer, Integer> befragmentDone;
+
+ // lock befragmentDone
+ private ReadWriteLock lock;
+
// use to merge profile from multi be
private List<Map<TNetworkAddress, List<RuntimeProfile>>> multiBeProfile = null;
@@ -231,8 +240,20 @@ public class ExecutionProfile {
public void markFragments(int fragments) {
profileFragmentDoneSignal = new MarkedCountDownLatch<>(fragments);
+ lock = new ReentrantReadWriteLock();
+ befragmentDone = new HashMap<>();
for (int fragmentId = 0; fragmentId < fragments; fragmentId++) {
profileFragmentDoneSignal.addMark(fragmentId, -1L /* value is meaningless */);
+ befragmentDone.put(fragmentId, 0);
+ }
+ }
+
+ public void addFragments(int fragmentId) {
+ lock.writeLock().lock();
+ try {
+ befragmentDone.put(fragmentId, befragmentDone.get(fragmentId) + 1);
+ } finally {
+ lock.writeLock().unlock();
}
}
@@ -282,8 +303,16 @@ public class ExecutionProfile {
public void markOneFragmentDone(int fragmentId) {
if (profileFragmentDoneSignal != null) {
- if (!profileFragmentDoneSignal.markedCountDown(fragmentId, -1L)) {
- LOG.warn("Mark fragment {} done failed", fragmentId);
+ lock.writeLock().lock();
+ try {
+ befragmentDone.put(fragmentId, befragmentDone.get(fragmentId) - 1);
+ if (befragmentDone.get(fragmentId) == 0) {
+ if (!profileFragmentDoneSignal.markedCountDown(fragmentId, -1L)) {
+ LOG.warn("Mark fragment {} done failed", fragmentId);
+ }
+ }
+ } finally {
+ lock.writeLock().unlock();
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 5e2d0ef85c8..165583b9e68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -3196,6 +3196,9 @@ public class Coordinator implements CoordInterface {
this.lastMissingHeartbeatTime = backend.getLastMissingHeartbeatTime();
this.enablePipelineX = enablePipelineX;
this.executionProfile = executionProfile;
+ if (enablePipelineX) {
+ executionProfile.addFragments(profileFragmentId);
+ }
}
public Stream<RuntimeProfile> profileStream() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org