You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/06 15:15:07 UTC
[doris] 05/36: [fix](memory) Fix query memory tracking #20253
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4ebf48b437b3a20255ad570c29471b661c26048e
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Mon Jun 5 08:33:38 2023 +0800
[fix](memory) Fix query memory tracking #20253
The memory released by the query end is recorded in the query mem tracker, main memory in _runtime_state.
fix page no cache memory tracking
Now the main reason for the inaccurate query memory tracking is that the virtual memory used by the query is sometimes much larger than the actual memory. And the mem hook counts virtual memory.
---
be/src/olap/page_cache.h | 2 --
be/src/olap/rowset/segment_v2/page_handle.h | 5 ++++-
be/src/pipeline/pipeline_fragment_context.cpp | 9 ++++++++-
be/src/runtime/plan_fragment_executor.cpp | 10 ++++++++--
4 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index cc52f4a252..7e5ca4de6e 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -42,7 +42,6 @@ public:
PageBase(size_t b) : _size(b), _capacity(b) {
_data = reinterpret_cast<char*>(TAllocator::alloc(_capacity, ALLOCATOR_ALIGNMENT_16));
- ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(_capacity);
}
PageBase(const PageBase&) = delete;
@@ -52,7 +51,6 @@ public:
if (_data != nullptr) {
DCHECK(_capacity != 0 && _size != 0);
TAllocator::free(_data, _capacity);
- ExecEnv::GetInstance()->page_no_cache_mem_tracker()->release(_capacity);
}
}
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h b/be/src/olap/rowset/segment_v2/page_handle.h
index 7e4f766524..6494048774 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -36,7 +36,9 @@ public:
// This class will take the ownership of input data's memory. It will
// free it when deconstructs.
- PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {}
+ PageHandle(DataPage* data) : _is_data_owner(true), _data(data) {
+ ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(_data->capacity());
+ }
// This class will take the content of cache data, and will make input
// cache_data to a invalid cache handle.
@@ -59,6 +61,7 @@ public:
~PageHandle() {
if (_is_data_owner) {
+ ExecEnv::GetInstance()->page_no_cache_mem_tracker()->release(_data->capacity());
delete _data;
} else {
DCHECK(_data == nullptr);
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index eea8d34b08..32207ad216 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -143,7 +143,14 @@ PipelineFragmentContext::PipelineFragmentContext(
}
PipelineFragmentContext::~PipelineFragmentContext() {
- _call_back(_runtime_state.get(), &_exec_status);
+ if (_runtime_state != nullptr) {
+ // The memory released by the query end is recorded in the query mem tracker, main memory in _runtime_state.
+ SCOPED_ATTACH_TASK(_runtime_state.get());
+ _call_back(_runtime_state.get(), &_exec_status);
+ _runtime_state.reset();
+ } else {
+ _call_back(_runtime_state.get(), &_exec_status);
+ }
DCHECK(!_report_thread_active);
}
diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp
index bd99a9a2e3..079719927b 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -88,7 +88,14 @@ PlanFragmentExecutor::PlanFragmentExecutor(ExecEnv* exec_env,
}
PlanFragmentExecutor::~PlanFragmentExecutor() {
- close();
+ if (_runtime_state != nullptr) {
+ // The memory released by the query end is recorded in the query mem tracker, main memory in _runtime_state.
+ SCOPED_ATTACH_TASK(_runtime_state.get());
+ close();
+ _runtime_state.reset();
+ } else {
+ close();
+ }
// at this point, the report thread should have been stopped
DCHECK(!_report_thread_active);
}
@@ -274,7 +281,6 @@ Status PlanFragmentExecutor::open() {
if (_cancel_reason == PPlanFragmentCancelReason::CALL_RPC_ERROR) {
status = Status::RuntimeError(_cancel_msg);
} else if (_cancel_reason == PPlanFragmentCancelReason::MEMORY_LIMIT_EXCEED) {
- // status = Status::MemoryAllocFailed(_cancel_msg);
status = Status::MemoryLimitExceeded(_cancel_msg);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org