You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/04/07 14:03:07 UTC
[doris] branch branch-1.2-lts updated: [enchancement](memory) tracking lru cache memory and page memory not in cache (#18361) (#18462)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new bfe3e60a5f [enchancement](memory) tracking lru cache memory and page memory not in cache (#18361) (#18462)
bfe3e60a5f is described below
commit bfe3e60a5f96b6e29f655a4b2c0f6c9c2fcfe9e4
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Fri Apr 7 22:02:59 2023 +0800
[enchancement](memory) tracking lru cache memory and page memory not in cache (#18361) (#18462)
Statistics lru cache memory in metrics
Statistics page memory not in cache in mem tracker
---
be/src/olap/lru_cache.cpp | 1 +
be/src/olap/lru_cache.h | 2 ++
be/src/olap/rowset/segment_v2/page_handle.h | 6 +++++-
be/src/olap/tablet_manager.cpp | 3 ++-
be/src/runtime/exec_env.h | 4 ++++
be/src/runtime/exec_env_init.cpp | 2 ++
be/src/runtime/memory/thread_mem_tracker_mgr.h | 20 ++++++++++++--------
be/src/util/doris_metrics.cpp | 2 ++
be/src/util/doris_metrics.h | 1 +
9 files changed, 31 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 81d9803ada..e89f520cae 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -304,6 +304,7 @@ Cache::Handle* LRUCache::insert(const CacheKey& key, uint32_t hash, void* value,
// The memory of the parameter value should be recorded in the tls mem tracker,
// transfer the memory ownership of the value to ShardedLRUCache::_mem_tracker.
THREAD_MEM_TRACKER_TRANSFER_TO(e->bytes, tracker);
+ DorisMetrics::instance()->lru_cache_memory_bytes->increment(e->bytes);
LRUHandle* to_remove_head = nullptr;
{
std::lock_guard<std::mutex> l(_mutex);
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 7d55d859c9..b8ddcb5fa9 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -16,6 +16,7 @@
#include "olap/olap_common.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/thread_context.h"
+#include "util/doris_metrics.h"
#include "util/metrics.h"
#include "util/slice.h"
@@ -260,6 +261,7 @@ typedef struct LRUHandle {
void free() {
(*deleter)(key(), value);
THREAD_MEM_TRACKER_TRANSFER_FROM(bytes, mem_tracker);
+ DorisMetrics::instance()->lru_cache_memory_bytes->increment(-bytes);
::free(this);
}
diff --git a/be/src/olap/rowset/segment_v2/page_handle.h b/be/src/olap/rowset/segment_v2/page_handle.h
index ecb5410a42..c2f82573c2 100644
--- a/be/src/olap/rowset/segment_v2/page_handle.h
+++ b/be/src/olap/rowset/segment_v2/page_handle.h
@@ -19,6 +19,7 @@
#include "gutil/macros.h" // for DISALLOW_COPY_AND_ASSIGN
#include "olap/page_cache.h"
+#include "runtime/exec_env.h"
#include "util/slice.h" // for Slice
namespace doris {
@@ -35,7 +36,9 @@ public:
// This class will take the ownership of input data's memory. It will
// free it when deconstructs.
- PageHandle(const Slice& data) : _is_data_owner(true), _data(data) {}
+ PageHandle(const Slice& data) : _is_data_owner(true), _data(data) {
+ ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(_data.size);
+ }
// This class will take the content of cache data, and will make input
// cache_data to a invalid cache handle.
@@ -61,6 +64,7 @@ public:
~PageHandle() {
if (_is_data_owner && _data.size > 0) {
delete[] _data.data;
+ ExecEnv::GetInstance()->page_no_cache_mem_tracker()->consume(-_data.size);
}
}
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 19fb4de676..fa7c25e9c4 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -76,7 +76,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_meta_mem_consumption, MetricUnit::BYTE
TabletManager::TabletManager(int32_t tablet_map_lock_shard_size)
: _mem_tracker(std::make_shared<MemTracker>(
"TabletManager", ExecEnv::GetInstance()->experimental_mem_tracker())),
- _tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta")),
+ _tablet_meta_mem_tracker(std::make_shared<MemTracker>(
+ "TabletMeta", ExecEnv::GetInstance()->experimental_mem_tracker())),
_tablets_shards_size(tablet_map_lock_shard_size),
_tablets_shards_mask(tablet_map_lock_shard_size - 1) {
CHECK_GT(_tablets_shards_size, 0);
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index f1bb540326..12ba99547b 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -120,10 +120,12 @@ public:
std::shared_ptr<MemTrackerLimiter> orphan_mem_tracker() { return _orphan_mem_tracker; }
MemTrackerLimiter* orphan_mem_tracker_raw() { return _orphan_mem_tracker_raw; }
MemTrackerLimiter* experimental_mem_tracker() { return _experimental_mem_tracker.get(); }
+ MemTracker* page_no_cache_mem_tracker() { return _page_no_cache_mem_tracker.get(); }
ThreadResourceMgr* thread_mgr() { return _thread_mgr; }
PriorityThreadPool* scan_thread_pool() { return _scan_thread_pool; }
PriorityThreadPool* remote_scan_thread_pool() { return _remote_scan_thread_pool; }
ThreadPool* limited_scan_thread_pool() { return _limited_scan_thread_pool.get(); }
+
ThreadPool* send_batch_thread_pool() { return _send_batch_thread_pool.get(); }
ThreadPool* download_cache_thread_pool() { return _download_cache_thread_pool.get(); }
ThreadPool* send_report_thread_pool() { return _send_report_thread_pool.get(); }
@@ -210,6 +212,8 @@ private:
std::shared_ptr<MemTrackerLimiter> _orphan_mem_tracker;
MemTrackerLimiter* _orphan_mem_tracker_raw;
std::shared_ptr<MemTrackerLimiter> _experimental_mem_tracker;
+ // page size not in cache, data page/index page/etc.
+ std::shared_ptr<MemTracker> _page_no_cache_mem_tracker;
// The following two thread pools are used in different scenarios.
// _scan_thread_pool is a priority thread pool.
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 5cdd5a0eef..4a7ad1d29b 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -308,6 +308,8 @@ void ExecEnv::init_mem_tracker() {
_orphan_mem_tracker_raw = _orphan_mem_tracker.get();
_experimental_mem_tracker = std::make_shared<MemTrackerLimiter>(
MemTrackerLimiter::Type::EXPERIMENTAL, "ExperimentalSet");
+ _page_no_cache_mem_tracker =
+ std::make_shared<MemTracker>("PageNoCache", _orphan_mem_tracker_raw);
}
void ExecEnv::init_download_cache_buf() {
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 5898f4c338..394a4c0993 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -85,11 +85,11 @@ public:
bool is_attach_query() { return _fragment_instance_id != TUniqueId(); }
std::shared_ptr<MemTrackerLimiter> limiter_mem_tracker() {
- if (!_init) init(); // ExecEnv not initialized when thread is created.
+ init();
return _limiter_tracker;
}
MemTrackerLimiter* limiter_mem_tracker_raw() {
- if (!_init) init();
+ init();
return _limiter_tracker_raw;
}
@@ -148,11 +148,15 @@ private:
};
inline void ThreadMemTrackerMgr::init() {
- DCHECK(_limiter_tracker == nullptr);
- _limiter_tracker = ExecEnv::GetInstance()->orphan_mem_tracker();
- _limiter_tracker_raw = ExecEnv::GetInstance()->orphan_mem_tracker_raw();
- _check_limit = true;
- _init = true;
+ // 1. Initialize in the thread context when the thread starts
+ // 2. ExecEnv not initialized when thread start, initialized in limiter_mem_tracker().
+ if (!_init) {
+ DCHECK(_limiter_tracker == nullptr);
+ _limiter_tracker = ExecEnv::GetInstance()->orphan_mem_tracker();
+ _limiter_tracker_raw = ExecEnv::GetInstance()->orphan_mem_tracker_raw();
+ _check_limit = true;
+ _init = true;
+ }
}
inline bool ThreadMemTrackerMgr::push_consumer_tracker(MemTracker* tracker) {
@@ -207,7 +211,7 @@ inline bool ThreadMemTrackerMgr::flush_untracked_mem() {
// Temporary memory may be allocated during the consumption of the mem tracker, which will lead to entering
// the Memory Hook again, so suspend consumption to avoid falling into an infinite loop.
_stop_consume = true;
- if (!_init) init(); // ExecEnv not initialized when thread is created.
+ init();
DCHECK(_limiter_tracker_raw);
old_untracked_mem = _untracked_mem;
if (_count_scope_mem) _scope_mem += _untracked_mem;
diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp
index f26d2aeb1c..f27b72f268 100644
--- a/be/src/util/doris_metrics.cpp
+++ b/be/src/util/doris_metrics.cpp
@@ -160,6 +160,7 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(blocks_open_writing, MetricUnit::BLOCKS);
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_memory_total_byte, MetricUnit::BYTES);
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_sql_total_count, MetricUnit::NOUNIT);
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_partition_total_count, MetricUnit::NOUNIT);
+DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(lru_cache_memory_bytes, MetricUnit::BYTES);
DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(upload_total_byte, MetricUnit::BYTES);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(upload_rowset_count, MetricUnit::ROWSETS);
@@ -289,6 +290,7 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_memory_total_byte);
INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_sql_total_count);
INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_partition_total_count);
+ INT_GAUGE_METRIC_REGISTER(_server_metric_entity, lru_cache_memory_bytes);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, local_file_reader_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, s3_file_reader_total);
diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h
index da02613000..10bcd28393 100644
--- a/be/src/util/doris_metrics.h
+++ b/be/src/util/doris_metrics.h
@@ -198,6 +198,7 @@ public:
UIntGauge* query_cache_memory_total_byte;
UIntGauge* query_cache_sql_total_count;
UIntGauge* query_cache_partition_total_count;
+ IntGauge* lru_cache_memory_bytes;
UIntGauge* scanner_thread_pool_queue_size;
UIntGauge* add_batch_task_queue_size;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org