You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/10/19 04:34:58 UTC
[doris] branch branch-1.1-lts updated: [branch-1.1-lts](memtracker) Fix mem hook cancel query #13396
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 5f73879c2c [branch-1.1-lts](memtracker) Fix mem hook cancel query #13396
5f73879c2c is described below
commit 5f73879c2c6188b459e21d019f33e88b857abb0e
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Wed Oct 19 12:34:52 2022 +0800
[branch-1.1-lts](memtracker) Fix mem hook cancel query #13396
Allow independent enable based on proc/mem_info check mem limit and cancel query
---
be/src/common/config.h | 5 ++++-
be/src/runtime/mem_tracker.h | 2 +-
be/src/runtime/memory/mem_tracker_limiter.h | 11 ++++++++---
be/src/runtime/memory/thread_mem_tracker_mgr.cpp | 2 +-
be/src/runtime/runtime_state.cpp | 2 +-
be/test/runtime/mem_limit_test.cpp | 8 ++++----
6 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index dfba339af5..132d7b43fd 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -632,7 +632,10 @@ CONF_Int16(mem_tracker_level, "0");
// Whether Hook TCmalloc new/delete, currently consume/release tls mem tracker in Hook.
CONF_Bool(enable_tcmalloc_hook, "true");
-CONF_Bool(enable_cancel_query, "false");
+// Cancel query if the process physical memory recorded in `/proc/mem_info` exceeds the limit.
+CONF_Bool(enable_proc_meminfo_cancel_query, "false");
+// Cancel query if query mem tracker exceeds the limit.
+CONF_Bool(enable_mem_tracker_cancel_query, "false");
// If true, switch TLS MemTracker to count more detailed memory,
// including caches such as ExecNode operators and TabletManager.
diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h
index e7a14c1ac7..d3808cdddb 100644
--- a/be/src/runtime/mem_tracker.h
+++ b/be/src/runtime/mem_tracker.h
@@ -173,7 +173,7 @@ public:
// This is independent of the consumption value of the mem tracker, which counts the virtual memory
// of the process malloc.
// for fast, expect MemInfo::initialized() to be true.
- if (PerfCounters::get_vm_rss() + bytes >= MemInfo::mem_limit() && config::enable_cancel_query) {
+ if (PerfCounters::get_vm_rss() + bytes >= MemInfo::mem_limit()) {
return Status::MemoryLimitExceeded(fmt::format(
"{}: TryConsume failed, bytes={} process whole consumption={} mem limit={}",
label_, bytes, MemInfo::current_mem(), MemInfo::mem_limit()));
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h
index df3d271494..001493bc80 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -80,7 +80,11 @@ public:
bytes >=
MemInfo::mem_limit() ||
PerfCounters::get_vm_rss() + bytes >= MemInfo::hard_mem_limit()) {
- return true;
+ if (config::enable_proc_meminfo_cancel_query) {
+ return true;
+ } else {
+ ExecEnv::GetInstance()->new_process_mem_tracker()->print_log_usage("sys_mem_exceed_limit_check");
+ }
}
return false;
}
@@ -306,7 +310,7 @@ inline bool MemTrackerLimiter::try_consume(int64_t bytes, std::string& failed_ms
MemTrackerLimiter* tracker = _all_ancestors[i];
// Process tracker does not participate in the process memory limit, process tracker consumption is virtual memory,
// and there is a diff between the real physical memory value of the process. It is replaced by check_sys_mem_info.
- if (tracker->limit() < 0 || tracker->label() == "Process") {
+ if (tracker->limit() < 0 || !config::enable_mem_tracker_cancel_query || tracker->label() == "Process") {
tracker->_consumption->add(bytes); // No limit at this tracker.
} else {
if (!tracker->_consumption->try_add(bytes, tracker->limit())) {
@@ -326,10 +330,11 @@ inline bool MemTrackerLimiter::try_consume(int64_t bytes, std::string& failed_ms
}
inline Status MemTrackerLimiter::check_limit(int64_t bytes) {
- if (bytes <= 0 || !config::enable_cancel_query) return Status::OK();
+ if (bytes <= 0) return Status::OK();
if (sys_mem_exceed_limit_check(bytes)) {
return Status::MemoryLimitExceeded(limit_exceeded_errmsg_sys_str(bytes));
}
+ if (!config::enable_mem_tracker_cancel_query) return Status::OK();
int i;
// Walk the tracker tree top-down.
for (i = _limited_ancestors.size() - 1; i >= 0; --i) {
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
index 7811fddeaa..3cb97c6ff8 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp
@@ -45,7 +45,7 @@ void ThreadMemTrackerMgr::detach_limiter_tracker() {
}
void ThreadMemTrackerMgr::exceeded_cancel_task(const std::string& cancel_details) {
- if (_fragment_instance_id_stack.back() != TUniqueId() && config::enable_cancel_query) {
+ if (_fragment_instance_id_stack.back() != TUniqueId()) {
ExecEnv::GetInstance()->fragment_mgr()->cancel(
_fragment_instance_id_stack.back(), PPlanFragmentCancelReason::MEMORY_LIMIT_EXCEED,
cancel_details);
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 864564e18b..fb3130e9d6 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -322,7 +322,7 @@ Status RuntimeState::set_mem_limit_exceeded(const std::string& msg) {
Status RuntimeState::check_query_state(const std::string& msg) {
// TODO: it would be nice if this also checked for cancellation, but doing so breaks
// cases where we use Status::Cancelled("Cancelled") to indicate that the limit was reached.
- if (config::enable_cancel_query) RETURN_IF_LIMIT_EXCEEDED(this, msg);
+ if (config::enable_mem_tracker_cancel_query) RETURN_IF_LIMIT_EXCEEDED(this, msg);
return query_status();
}
diff --git a/be/test/runtime/mem_limit_test.cpp b/be/test/runtime/mem_limit_test.cpp
index cc03e0b84f..903d112454 100644
--- a/be/test/runtime/mem_limit_test.cpp
+++ b/be/test/runtime/mem_limit_test.cpp
@@ -24,7 +24,7 @@
namespace doris {
TEST(MemTrackerTest, SingleTrackerNoLimit) {
- config::enable_cancel_query = true;
+ config::enable_mem_tracker_cancel_query = true;
auto t = MemTracker::CreateTracker();
EXPECT_FALSE(t->has_limit());
t->Consume(10);
@@ -38,7 +38,7 @@ TEST(MemTrackerTest, SingleTrackerNoLimit) {
}
TEST(MemTestTest, SingleTrackerWithLimit) {
- config::enable_cancel_query = true;
+ config::enable_mem_tracker_cancel_query = true;
auto t = MemTracker::CreateTracker(11, "limit tracker");
EXPECT_TRUE(t->has_limit());
t->Consume(10);
@@ -54,7 +54,7 @@ TEST(MemTestTest, SingleTrackerWithLimit) {
}
TEST(MemTestTest, TrackerHierarchy) {
- config::enable_cancel_query = true;
+ config::enable_mem_tracker_cancel_query = true;
auto p = MemTracker::CreateTracker(100);
auto c1 = MemTracker::CreateTracker(80, "c1", p);
auto c2 = MemTracker::CreateTracker(50, "c2", p);
@@ -98,7 +98,7 @@ TEST(MemTestTest, TrackerHierarchy) {
}
TEST(MemTestTest, TrackerHierarchyTryConsume) {
- config::enable_cancel_query = true;
+ config::enable_mem_tracker_cancel_query = true;
auto p = MemTracker::CreateTracker(100);
auto c1 = MemTracker::CreateTracker(80, "c1", p);
auto c2 = MemTracker::CreateTracker(50, "c2", p);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org