You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mr...@apache.org on 2026/04/24 03:16:37 UTC
(doris) branch master updated: [fix](be) Refine revocable memory accounting for spill (#62581)
This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7dbb507e379 [fix](be) Refine revocable memory accounting for spill (#62581)
7dbb507e379 is described below
commit 7dbb507e379344ab2741d2979a2ff13dde2f08fb
Author: Jerry Hu <hu...@selectdb.com>
AuthorDate: Fri Apr 24 11:16:30 2026 +0800
[fix](be) Refine revocable memory accounting for spill (#62581)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: Exclude small non-spillable revocable buffers from
pipeline task revocable memory accounting and handle queries without
revocable tasks when triggering memory revocation.
### Release note
None
### Check List (For Author)
- Test: No need to test (commit existing staged changes only)
- Behavior changed: Yes (revocable memory estimation and empty
revocation handling are adjusted)
- Does this need documentation: No
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/exec/pipeline/pipeline_task.cpp | 34 ++++++++++++----------
be/src/exec/pipeline/pipeline_task.h | 1 +
.../workload_management/query_task_controller.cpp | 16 +++++++++-
3 files changed, 35 insertions(+), 16 deletions(-)
diff --git a/be/src/exec/pipeline/pipeline_task.cpp b/be/src/exec/pipeline/pipeline_task.cpp
index 146d3f028c6..b98bdae81de 100644
--- a/be/src/exec/pipeline/pipeline_task.cpp
+++ b/be/src/exec/pipeline/pipeline_task.cpp
@@ -418,14 +418,7 @@ bool PipelineTask::_should_trigger_revoking(const size_t reserve_size) const {
}
if (is_high_memory_pressure) {
- const auto revocable_size = [&]() {
- size_t total = _sink->revocable_mem_size(_state);
- for (const auto& op : _operators) {
- total += op->revocable_mem_size(_state);
- }
- return total;
- }();
-
+ const auto revocable_size = _get_revocable_size();
const auto total_estimated_revocable = revocable_size * parallelism;
return total_estimated_revocable >= int64_t(double(query_limit) * 0.2);
}
@@ -1007,20 +1000,31 @@ std::string PipelineTask::debug_string() {
return fmt::to_string(debug_string_buffer);
}
-size_t PipelineTask::get_revocable_size() const {
- if (!_opened || is_finalized() || _running || (_eos && !_spilling)) {
- return 0;
- }
-
+size_t PipelineTask::_get_revocable_size() const {
// Sum revocable memory from every operator in the pipeline + the sink.
// Each operator reports only its own revocable memory (no child recursion).
- size_t total = _sink->revocable_mem_size(_state);
+ size_t total = 0;
+ size_t sink_revocable_size = _sink->revocable_mem_size(_state);
+ if (sink_revocable_size >= SpillFile::MIN_SPILL_WRITE_BATCH_MEM) {
+ total += sink_revocable_size;
+ }
for (const auto& op : _operators) {
- total += op->revocable_mem_size(_state);
+ size_t ops_revocable_size = op->revocable_mem_size(_state);
+ if (ops_revocable_size >= SpillFile::MIN_SPILL_WRITE_BATCH_MEM) {
+ total += ops_revocable_size;
+ }
}
return total;
}
+size_t PipelineTask::get_revocable_size() const {
+ if (!_opened || is_finalized() || _running || (_eos && !_spilling)) {
+ return 0;
+ }
+
+ return _get_revocable_size();
+}
+
Status PipelineTask::revoke_memory(const std::shared_ptr<SpillContext>& spill_context) {
DCHECK(spill_context);
if (is_finalized()) {
diff --git a/be/src/exec/pipeline/pipeline_task.h b/be/src/exec/pipeline/pipeline_task.h
index 280290799e3..bb1440bfcb7 100644
--- a/be/src/exec/pipeline/pipeline_task.h
+++ b/be/src/exec/pipeline/pipeline_task.h
@@ -204,6 +204,7 @@ private:
// otherwise return true.
bool _try_to_reserve_memory(const size_t reserve_size, OperatorBase* op);
bool _should_trigger_revoking(const size_t reserve_size) const;
+ size_t _get_revocable_size() const;
const TUniqueId _query_id;
const uint32_t _index;
diff --git a/be/src/runtime/workload_management/query_task_controller.cpp b/be/src/runtime/workload_management/query_task_controller.cpp
index d9210c8bd17..a7bc5b5fd78 100644
--- a/be/src/runtime/workload_management/query_task_controller.cpp
+++ b/be/src/runtime/workload_management/query_task_controller.cpp
@@ -17,6 +17,8 @@
#include "runtime/workload_management/query_task_controller.h"
+#include <algorithm>
+
#include "exec/pipeline/pipeline_fragment_context.h"
#include "runtime/query_context.h"
#include "runtime/workload_management/task_controller.h"
@@ -149,7 +151,19 @@ Status QueryTaskController::revoke_memory() {
fragments.emplace_back(std::move(fragment_ctx));
}
- std::sort(tasks.begin(), tasks.end(), [](auto&& l, auto&& r) { return l.first > r.first; });
+ if (tasks.empty()) {
+ LOG(INFO) << fmt::format(
+ "Query {} try to revoke memory, but there is no revocable task, maybe because the "
+ "query was spilled already. Query memory usage: {}, wg info: {}. {}",
+ print_id(query_ctx->query_id()),
+ PrettyPrinter::print_bytes(query_ctx->query_mem_tracker()->consumption()),
+ query_ctx->workload_group()->memory_debug_string(),
+ doris::ProcessProfile::instance()->memory_profile()->process_memory_detail_str());
+ query_ctx->set_memory_sufficient(true);
+ return Status::OK();
+ }
+
+ std::ranges::sort(tasks, [](auto&& l, auto&& r) { return l.first > r.first; });
// Do not use memlimit, use current memory usage.
// For example, if current limit is 1.6G, but current used is 1G, if reserve failed
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org