You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/12/03 03:27:50 UTC
[doris] branch branch-1.1-lts updated: [fix](tcmalloc_gc) optimize policy of tcmalloc gc (#14777)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new d132483e5d [fix](tcmalloc_gc) optimize policy of tcmalloc gc (#14777)
d132483e5d is described below
commit d132483e5d20a62d06d532301269e8c56747f428
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Sat Dec 3 11:27:44 2022 +0800
[fix](tcmalloc_gc) optimize policy of tcmalloc gc (#14777)
Release memory when memory pressure is above pressure limit and
keep at lease 2% memory as tcmalloc cache.
---
be/src/common/daemon.cpp | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index da85028bb5..976e334d8b 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -114,8 +114,10 @@ void Daemon::tcmalloc_gc_thread() {
(int64_t)tc_cached_bytes - (tc_used_bytes * max_cache_percent / 100);
int64_t memory_pressure = 0;
+ int64_t memory_pressure_no_cache = 0;
int64_t alloc_bytes = std::max(rss, tc_alloc_bytes);
memory_pressure = alloc_bytes * 100 / physical_limit_bytes;
+ memory_pressure_no_cache = (alloc_bytes - tc_cached_bytes) * 100 / physical_limit_bytes;
expected_aggressive_decommit = init_aggressive_decommit;
if (memory_pressure > pressure_limit) {
@@ -123,13 +125,13 @@ void Daemon::tcmalloc_gc_thread() {
// Ideally, we should reuse cache and not allocate from system any more,
// however, it is hard to set limit on cache of tcmalloc and doris
// use mmap in vectorized mode.
- if (last_memory_pressure <= pressure_limit) {
- int64_t min_free_bytes = alloc_bytes - physical_limit_bytes * 9 / 10;
- to_free_bytes = std::max(to_free_bytes, min_free_bytes);
- to_free_bytes = std::max(to_free_bytes, tc_cached_bytes * 30 / 100);
- to_free_bytes = std::min(to_free_bytes, tc_cached_bytes);
- expected_aggressive_decommit = 1;
- }
+ // Limit cache capactiy is enough.
+ int64_t min_free_bytes = alloc_bytes - physical_limit_bytes * 9 / 10;
+ to_free_bytes = std::max(to_free_bytes, min_free_bytes);
+ to_free_bytes = std::max(to_free_bytes, tc_cached_bytes * 30 / 100);
+ // We assure that we have at least 500M bytes in cache.
+ to_free_bytes = std::min(to_free_bytes, tc_cached_bytes - 500 * 1024 * 1024);
+ expected_aggressive_decommit = (memory_pressure_no_cache > pressure_limit);
last_ms = kMaxLastMs;
} else if (memory_pressure > (pressure_limit - 10)) {
// In most cases, adjusting release rate is enough, if memory are consumed quickly
@@ -155,7 +157,8 @@ void Daemon::tcmalloc_gc_thread() {
}
last_memory_pressure = memory_pressure;
- if (to_free_bytes > 0) {
+ // We release at least 2% bytes once, frequent releasing hurts performance.
+ if (to_free_bytes > physical_limit_bytes * 2 / 100) {
last_ms += kIntervalMs;
if (last_ms >= kMaxLastMs) {
LOG(INFO) << "generic.current_allocated_bytes " << tc_used_bytes
@@ -169,7 +172,6 @@ void Daemon::tcmalloc_gc_thread() {
last_ms = 0;
}
} else {
- DCHECK(tc_cached_bytes <= tc_used_bytes * max_cache_percent / 100);
last_ms = 0;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org