[doris] branch master updated: [fix] (mem tracker) Fix BE hangs at startup, stuck in tcmalloc hook call ExecEnv::GetInstance() (#10517)

[zo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 0d99b27d89 [fix] (mem tracker) Fix BE hangs at startup, stuck in tcmalloc hook call ExecEnv::GetInstance() (#10517)
0d99b27d89 is described below

commit 0d99b27d893ed660a1618c0f6d5d3e75785385b4
Author: Kidd <10...@users.noreply.github.com>
AuthorDate: Fri Jul 1 10:20:53 2022 +0800

    [fix] (mem tracker) Fix BE hangs at startup, stuck in tcmalloc hook call ExecEnv::GetInstance() (#10517)
    
    1. Added flag exec_env_existed to indicate whether ExecEnv Instance is created.
    2. ThreadMemTrackerMgr::add_tracker fail when USE_MEM_TRACKER=OFF, add USE_MEM_TRACKER compile option.
---
 be/src/runtime/exec_env.h               | 3 +++
 be/src/runtime/tcmalloc_hook.h          | 8 ++++----
 be/src/runtime/thread_context.h         | 8 ++------
 be/src/runtime/thread_mem_tracker_mgr.h | 2 ++
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index e0e09ae047..03e9971152 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -72,6 +72,8 @@ class ClientCache;
 
 class HeartbeatFlags;
 
+static bool exec_env_existed = false;
+
 // Execution environment for queries/plan fragments.
 // Contains all required global structures, and handles to
 // singleton services. Clients must call StartServices exactly
@@ -87,6 +89,7 @@ public:
     /// we return the most recently created instance.
     static ExecEnv* GetInstance() {
         static ExecEnv s_exec_env;
+        exec_env_existed = true;
         return &s_exec_env;
     }
 
diff --git a/be/src/runtime/tcmalloc_hook.h b/be/src/runtime/tcmalloc_hook.h
index af60cdc178..dd0e1c788f 100644
--- a/be/src/runtime/tcmalloc_hook.h
+++ b/be/src/runtime/tcmalloc_hook.h
@@ -37,17 +37,17 @@
 //  destructor to control the behavior of consume can lead to unexpected behavior,
 //  like this: if (LIKELY(doris::start_thread_mem_tracker)) {
 void new_hook(const void* ptr, size_t size) {
-    if (doris::tls_ctx()) {
+    if (doris::thread_local_ctx._init) {
         doris::tls_ctx()->consume_mem(tc_nallocx(size, 0));
-    } else if (doris::ExecEnv::GetInstance()->initialized()) {
+    } else if (doris::exec_env_existed && doris::ExecEnv::GetInstance()->initialized()) {
         doris::MemTracker::get_process_tracker()->consume(tc_nallocx(size, 0));
     }
 }
 
 void delete_hook(const void* ptr) {
-    if (doris::tls_ctx()) {
+    if (doris::thread_local_ctx._init) {
         doris::tls_ctx()->release_mem(tc_malloc_size(const_cast<void*>(ptr)));
-    } else if (doris::ExecEnv::GetInstance()->initialized()) {
+    } else if (doris::exec_env_existed && doris::ExecEnv::GetInstance()->initialized()) {
         doris::MemTracker::get_process_tracker()->release(tc_malloc_size(const_cast<void*>(ptr)));
     }
 }
diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
index 33478b19c5..bf347e0a28 100644
--- a/be/src/runtime/thread_context.h
+++ b/be/src/runtime/thread_context.h
@@ -214,6 +214,7 @@ public:
 
     ThreadContext* get();
 
+    // TCMalloc hook is triggered during ThreadContext construction, which may lead to deadlock.
     bool _init = false;
 
 private:
@@ -227,12 +228,7 @@ static ThreadContext* tls_ctx() {
     if (tls != nullptr) {
         return tls;
     } else {
-        if (thread_local_ctx._init) {
-            return thread_local_ctx.get();
-        } else {
-            // TCMalloc hook is triggered during ThreadContext construction, which may lead to deadlock.
-            return nullptr;
-        }
+        return thread_local_ctx.get();
     }
 }
 
diff --git a/be/src/runtime/thread_mem_tracker_mgr.h b/be/src/runtime/thread_mem_tracker_mgr.h
index d8042ac0fa..ffc6fdc01f 100644
--- a/be/src/runtime/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/thread_mem_tracker_mgr.h
@@ -292,11 +292,13 @@ inline void ThreadMemTrackerMgr::noncache_try_consume(int64_t size) {
 }
 
 inline void ThreadMemTrackerMgr::add_tracker(const std::shared_ptr<MemTracker>& mem_tracker) {
+#ifdef USE_MEM_TRACKER
     DCHECK(_mem_trackers.find(mem_tracker->id()) == _mem_trackers.end()) << print_debug_string();
     _mem_trackers[mem_tracker->id()] = mem_tracker;
     DCHECK(_mem_trackers[mem_tracker->id()]) << print_debug_string();
     _untracked_mems[mem_tracker->id()] = 0;
     _mem_tracker_labels[mem_tracker->id()] = mem_tracker->label();
+#endif
 }
 
 inline std::shared_ptr<MemTracker> ThreadMemTrackerMgr::mem_tracker() {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org