You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/01/14 09:33:24 UTC

(doris) branch master updated: [fix](pipelineX) fix multi be may be missing profiles #29914

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 67ef9e304e3 [fix](pipelineX) fix multi be may be missing profiles #29914
67ef9e304e3 is described below

commit 67ef9e304e3cf4e9ad4042f408b87e8d0112317d
Author: Mryange <59...@users.noreply.github.com>
AuthorDate: Sun Jan 14 17:33:17 2024 +0800

    [fix](pipelineX) fix multi be may be missing profiles #29914
---
 .../doris/common/profile/ExecutionProfile.java     | 33 ++++++++++++++++++++--
 .../main/java/org/apache/doris/qe/Coordinator.java |  3 ++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index 9584a6f1fa1..24bd2355c56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -33,11 +33,14 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * ExecutionProfile is used to collect profile of a complete query plan(including query or load).
@@ -72,6 +75,12 @@ public class ExecutionProfile {
     // fragmentId -> dummy value
     private MarkedCountDownLatch<Integer, Long> profileFragmentDoneSignal;
 
+    // fragmentId -> The number of BE without 'done.
+    private Map<Integer, Integer> befragmentDone;
+
+    // lock befragmentDone
+    private ReadWriteLock lock;
+
     // use to merge profile from multi be
     private List<Map<TNetworkAddress, List<RuntimeProfile>>> multiBeProfile = null;
 
@@ -231,8 +240,20 @@ public class ExecutionProfile {
 
     public void markFragments(int fragments) {
         profileFragmentDoneSignal = new MarkedCountDownLatch<>(fragments);
+        lock = new ReentrantReadWriteLock();
+        befragmentDone = new HashMap<>();
         for (int fragmentId = 0; fragmentId < fragments; fragmentId++) {
             profileFragmentDoneSignal.addMark(fragmentId, -1L /* value is meaningless */);
+            befragmentDone.put(fragmentId, 0);
+        }
+    }
+
+    public void addFragments(int fragmentId) {
+        lock.writeLock().lock();
+        try {
+            befragmentDone.put(fragmentId, befragmentDone.get(fragmentId) + 1);
+        } finally {
+            lock.writeLock().unlock();
         }
     }
 
@@ -282,8 +303,16 @@ public class ExecutionProfile {
 
     public void markOneFragmentDone(int fragmentId) {
         if (profileFragmentDoneSignal != null) {
-            if (!profileFragmentDoneSignal.markedCountDown(fragmentId, -1L)) {
-                LOG.warn("Mark fragment {} done failed", fragmentId);
+            lock.writeLock().lock();
+            try {
+                befragmentDone.put(fragmentId, befragmentDone.get(fragmentId) - 1);
+                if (befragmentDone.get(fragmentId) == 0) {
+                    if (!profileFragmentDoneSignal.markedCountDown(fragmentId, -1L)) {
+                        LOG.warn("Mark fragment {} done failed", fragmentId);
+                    }
+                }
+            } finally {
+                lock.writeLock().unlock();
             }
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 5e2d0ef85c8..165583b9e68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -3196,6 +3196,9 @@ public class Coordinator implements CoordInterface {
             this.lastMissingHeartbeatTime = backend.getLastMissingHeartbeatTime();
             this.enablePipelineX = enablePipelineX;
             this.executionProfile = executionProfile;
+            if (enablePipelineX) {
+                executionProfile.addFragments(profileFragmentId);
+            }
         }
 
         public Stream<RuntimeProfile> profileStream() {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org