You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mr...@apache.org on 2026/04/24 03:16:37 UTC

(doris) branch master updated: [fix](be) Refine revocable memory accounting for spill (#62581)

This is an automated email from the ASF dual-hosted git repository.

mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 7dbb507e379 [fix](be) Refine revocable memory accounting for spill (#62581)
7dbb507e379 is described below

commit 7dbb507e379344ab2741d2979a2ff13dde2f08fb
Author: Jerry Hu <hu...@selectdb.com>
AuthorDate: Fri Apr 24 11:16:30 2026 +0800

    [fix](be) Refine revocable memory accounting for spill (#62581)
    
    ### What problem does this PR solve?
    
    Issue Number: None
    
    Related PR: None
    
    Problem Summary: Exclude small non-spillable revocable buffers from
    pipeline task revocable memory accounting and handle queries without
    revocable tasks when triggering memory revocation.
    
    ### Release note
    
    None
    
    ### Check List (For Author)
    
    - Test: No need to test (commit existing staged changes only)
    
    - Behavior changed: Yes (revocable memory estimation and empty
    revocation handling are adjusted)
    
    - Does this need documentation: No
    
    ### What problem does this PR solve?
    
    Issue Number: close #xxx
    
    Related PR: #xxx
    
    Problem Summary:
    
    ### Release note
    
    None
    
    ### Check List (For Author)
    
    - Test <!-- At least one of them must be included. -->
        - [ ] Regression test
        - [ ] Unit Test
        - [ ] Manual test (add detailed scripts or steps below)
        - [ ] No need to test or manual test. Explain why:
    - [ ] This is a refactor/code format and no logic has been changed.
            - [ ] Previous test can cover this change.
            - [ ] No code files have been changed.
            - [ ] Other reason <!-- Add your reason?  -->
    
    - Behavior changed:
        - [ ] No.
        - [ ] Yes. <!-- Explain the behavior change -->
    
    - Does this need documentation?
        - [ ] No.
    - [ ] Yes. <!-- Add document PR link here. eg:
    https://github.com/apache/doris-website/pull/1214 -->
    
    ### Check List (For Reviewer who merge this PR)
    
    - [ ] Confirm the release note
    - [ ] Confirm test cases
    - [ ] Confirm document
    - [ ] Add branch pick label <!-- Add branch pick label that this PR
    should merge into -->
---
 be/src/exec/pipeline/pipeline_task.cpp             | 34 ++++++++++++----------
 be/src/exec/pipeline/pipeline_task.h               |  1 +
 .../workload_management/query_task_controller.cpp  | 16 +++++++++-
 3 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/be/src/exec/pipeline/pipeline_task.cpp b/be/src/exec/pipeline/pipeline_task.cpp
index 146d3f028c6..b98bdae81de 100644
--- a/be/src/exec/pipeline/pipeline_task.cpp
+++ b/be/src/exec/pipeline/pipeline_task.cpp
@@ -418,14 +418,7 @@ bool PipelineTask::_should_trigger_revoking(const size_t reserve_size) const {
     }
 
     if (is_high_memory_pressure) {
-        const auto revocable_size = [&]() {
-            size_t total = _sink->revocable_mem_size(_state);
-            for (const auto& op : _operators) {
-                total += op->revocable_mem_size(_state);
-            }
-            return total;
-        }();
-
+        const auto revocable_size = _get_revocable_size();
         const auto total_estimated_revocable = revocable_size * parallelism;
         return total_estimated_revocable >= int64_t(double(query_limit) * 0.2);
     }
@@ -1007,20 +1000,31 @@ std::string PipelineTask::debug_string() {
     return fmt::to_string(debug_string_buffer);
 }
 
-size_t PipelineTask::get_revocable_size() const {
-    if (!_opened || is_finalized() || _running || (_eos && !_spilling)) {
-        return 0;
-    }
-
+size_t PipelineTask::_get_revocable_size() const {
     // Sum revocable memory from every operator in the pipeline + the sink.
     // Each operator reports only its own revocable memory (no child recursion).
-    size_t total = _sink->revocable_mem_size(_state);
+    size_t total = 0;
+    size_t sink_revocable_size = _sink->revocable_mem_size(_state);
+    if (sink_revocable_size >= SpillFile::MIN_SPILL_WRITE_BATCH_MEM) {
+        total += sink_revocable_size;
+    }
     for (const auto& op : _operators) {
-        total += op->revocable_mem_size(_state);
+        size_t ops_revocable_size = op->revocable_mem_size(_state);
+        if (ops_revocable_size >= SpillFile::MIN_SPILL_WRITE_BATCH_MEM) {
+            total += ops_revocable_size;
+        }
     }
     return total;
 }
 
+size_t PipelineTask::get_revocable_size() const {
+    if (!_opened || is_finalized() || _running || (_eos && !_spilling)) {
+        return 0;
+    }
+
+    return _get_revocable_size();
+}
+
 Status PipelineTask::revoke_memory(const std::shared_ptr<SpillContext>& spill_context) {
     DCHECK(spill_context);
     if (is_finalized()) {
diff --git a/be/src/exec/pipeline/pipeline_task.h b/be/src/exec/pipeline/pipeline_task.h
index 280290799e3..bb1440bfcb7 100644
--- a/be/src/exec/pipeline/pipeline_task.h
+++ b/be/src/exec/pipeline/pipeline_task.h
@@ -204,6 +204,7 @@ private:
     // otherwise return true.
     bool _try_to_reserve_memory(const size_t reserve_size, OperatorBase* op);
     bool _should_trigger_revoking(const size_t reserve_size) const;
+    size_t _get_revocable_size() const;
 
     const TUniqueId _query_id;
     const uint32_t _index;
diff --git a/be/src/runtime/workload_management/query_task_controller.cpp b/be/src/runtime/workload_management/query_task_controller.cpp
index d9210c8bd17..a7bc5b5fd78 100644
--- a/be/src/runtime/workload_management/query_task_controller.cpp
+++ b/be/src/runtime/workload_management/query_task_controller.cpp
@@ -17,6 +17,8 @@
 
 #include "runtime/workload_management/query_task_controller.h"
 
+#include <algorithm>
+
 #include "exec/pipeline/pipeline_fragment_context.h"
 #include "runtime/query_context.h"
 #include "runtime/workload_management/task_controller.h"
@@ -149,7 +151,19 @@ Status QueryTaskController::revoke_memory() {
         fragments.emplace_back(std::move(fragment_ctx));
     }
 
-    std::sort(tasks.begin(), tasks.end(), [](auto&& l, auto&& r) { return l.first > r.first; });
+    if (tasks.empty()) {
+        LOG(INFO) << fmt::format(
+                "Query {} try to revoke memory, but there is no revocable task, maybe because the "
+                "query was spilled already. Query memory usage: {}, wg info: {}. {}",
+                print_id(query_ctx->query_id()),
+                PrettyPrinter::print_bytes(query_ctx->query_mem_tracker()->consumption()),
+                query_ctx->workload_group()->memory_debug_string(),
+                doris::ProcessProfile::instance()->memory_profile()->process_memory_detail_str());
+        query_ctx->set_memory_sufficient(true);
+        return Status::OK();
+    }
+
+    std::ranges::sort(tasks, [](auto&& l, auto&& r) { return l.first > r.first; });
 
     // Do not use memlimit, use current memory usage.
     // For example, if current limit is 1.6G, but current used is 1G, if reserve failed


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org