(doris) branch spill_and_reserve updated: [revokememory] remove old revoke memory logic

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/spill_and_reserve by this push:
     new bf03ec25a70 [revokememory] remove old revoke memory logic
bf03ec25a70 is described below

commit bf03ec25a70d1fdc55e558b0535196a5cf777c00
Author: yiguolei <yi...@gmail.com>
AuthorDate: Thu Sep 5 15:11:57 2024 +0800

    [revokememory] remove old revoke memory logic
---
 be/src/pipeline/pipeline_task.cpp | 71 +--------------------------------------
 1 file changed, 1 insertion(+), 70 deletions(-)

diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 4e795a47f60..64b7997101a 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -352,16 +352,6 @@ Status PipelineTask::execute(bool* eos) {
         _block->clear_column_data(_root->row_desc().num_materialized_slots());
         auto* block = _block.get();
 
-        auto sink_revocable_mem_size = _sink->revocable_mem_size(_state);
-
-        bool is_wg_mem_low_water_mark = false;
-        bool is_wg_mem_high_water_mark = false;
-        if (should_revoke_memory(_state, sink_revocable_mem_size, is_wg_mem_low_water_mark,
-                                 is_wg_mem_high_water_mark)) {
-            RETURN_IF_ERROR(_sink->revoke_memory(_state));
-            continue;
-        }
-
         *eos = _eos;
         DBUG_EXECUTE_IF("fault_inject::PipelineXTask::executing", {
             Status status =
@@ -393,9 +383,7 @@ Status PipelineTask::execute(bool* eos) {
                                << ", try to reserve: " << reserve_size << "(sink reserve size:("
                                << sink_reserve_size << " )"
                                << ", sink name: " << _sink->get_name()
-                               << ", node id: " << _sink->node_id()
-                               << ", is_wg_mem_high_water_mark: " << is_wg_mem_high_water_mark
-                               << " failed: " << st.to_string()
+                               << ", node id: " << _sink->node_id() << " failed: " << st.to_string()
                                << ", debug info: " << GlobalMemoryArbitrator::process_mem_log_str();
                     {
                         _memory_sufficient_dependency->block();
@@ -446,63 +434,6 @@ Status PipelineTask::execute(bool* eos) {
     return Status::OK();
 }
 
-bool PipelineTask::should_revoke_memory(RuntimeState* state, int64_t revocable_mem_bytes,
-                                        bool& is_wg_mem_low_water_mark,
-                                        bool& is_wg_mem_high_water_mark) {
-    auto* query_ctx = state->get_query_ctx();
-    auto wg = query_ctx->workload_group();
-    if (!wg) {
-        LOG_ONCE(INFO) << "no workload group for query " << print_id(state->query_id());
-        return false;
-    }
-    const auto min_revocable_mem_bytes = state->min_revocable_mem();
-
-    if (UNLIKELY(state->enable_force_spill())) {
-        if (revocable_mem_bytes >= min_revocable_mem_bytes) {
-            LOG_ONCE(INFO) << "spill force, query: " << print_id(state->query_id());
-            return true;
-        }
-    }
-
-    wg->check_mem_used(&is_wg_mem_low_water_mark, &is_wg_mem_high_water_mark);
-    if (is_wg_mem_high_water_mark) {
-        if (revocable_mem_bytes > min_revocable_mem_bytes) {
-            VLOG_DEBUG << "query " << print_id(state->query_id())
-                       << " revoke memory, hight water mark";
-            return true;
-        }
-        return false;
-    } else if (is_wg_mem_low_water_mark) {
-        int64_t spill_threshold = query_ctx->spill_threshold();
-        int64_t memory_usage = query_ctx->query_mem_tracker->consumption();
-        if (spill_threshold == 0 || memory_usage < spill_threshold) {
-            return false;
-        }
-        auto big_memory_operator_num = query_ctx->get_running_big_mem_op_num();
-        DCHECK(big_memory_operator_num >= 0);
-        int64_t mem_limit_of_op;
-        if (0 == big_memory_operator_num) {
-            return false;
-        } else {
-            mem_limit_of_op = spill_threshold / big_memory_operator_num;
-        }
-
-        LOG_EVERY_T(INFO, 1) << "query " << print_id(state->query_id())
-                             << " revoke memory, low water mark, revocable_mem_bytes: "
-                             << PrettyPrinter::print_bytes(revocable_mem_bytes)
-                             << ", mem_limit_of_op: " << PrettyPrinter::print_bytes(mem_limit_of_op)
-                             << ", min_revocable_mem_bytes: "
-                             << PrettyPrinter::print_bytes(min_revocable_mem_bytes)
-                             << ", memory_usage: " << PrettyPrinter::print_bytes(memory_usage)
-                             << ", spill_threshold: " << PrettyPrinter::print_bytes(spill_threshold)
-                             << ", big_memory_operator_num: " << big_memory_operator_num;
-        return (revocable_mem_bytes > mem_limit_of_op ||
-                revocable_mem_bytes > min_revocable_mem_bytes);
-    } else {
-        return false;
-    }
-}
-
 void PipelineTask::finalize() {
     std::unique_lock<std::mutex> lc(_dependency_lock);
     _finalized = true;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org