You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zo...@apache.org on 2022/12/04 02:51:04 UTC
[doris] branch branch-1.1-lts updated: [fix](tcmalloc) consider rss when free cached memory (#14782)
This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new ed5d8b2eb4 [fix](tcmalloc) consider rss when free cached memory (#14782)
ed5d8b2eb4 is described below
commit ed5d8b2eb4cb6841f9b95219aa12c1b086c09207
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Sun Dec 4 10:50:58 2022 +0800
[fix](tcmalloc) consider rss when free cached memory (#14782)
If rss usage is low, we do not free cached memory aggressively.
---
be/src/common/daemon.cpp | 31 +++++++++++++++++--------------
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index 976e334d8b..72a3520226 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -76,7 +76,7 @@ void Daemon::tcmalloc_gc_thread() {
double release_rates[10] = {1.0, 1.0, 1.0, 5.0, 5.0, 20.0, 50.0, 100.0, 500.0, 2000.0};
int64_t pressure_limit = 90;
bool is_performance_mode = false;
- size_t physical_limit_bytes = std::min(MemInfo::hard_mem_limit(), MemInfo::mem_limit());
+ int64_t physical_limit_bytes = std::min(MemInfo::hard_mem_limit(), MemInfo::mem_limit());
if (config::memory_mode == std::string("performance")) {
max_cache_percent = 100;
@@ -109,15 +109,16 @@ void Daemon::tcmalloc_gc_thread() {
&tc_alloc_bytes);
MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes",
&tc_used_bytes);
- int64_t tc_cached_bytes = tc_alloc_bytes - tc_used_bytes;
+ int64_t tc_cached_bytes = (int64_t)tc_alloc_bytes - (int64_t)tc_used_bytes;
int64_t to_free_bytes =
- (int64_t)tc_cached_bytes - (tc_used_bytes * max_cache_percent / 100);
+ (int64_t)tc_cached_bytes - ((int64_t)tc_used_bytes * max_cache_percent / 100);
+ to_free_bytes = std::max(to_free_bytes, (int64_t)0);
int64_t memory_pressure = 0;
- int64_t memory_pressure_no_cache = 0;
+ int64_t rss_pressure = 0;
int64_t alloc_bytes = std::max(rss, tc_alloc_bytes);
memory_pressure = alloc_bytes * 100 / physical_limit_bytes;
- memory_pressure_no_cache = (alloc_bytes - tc_cached_bytes) * 100 / physical_limit_bytes;
+ rss_pressure = rss * 100 / physical_limit_bytes;
expected_aggressive_decommit = init_aggressive_decommit;
if (memory_pressure > pressure_limit) {
@@ -126,12 +127,14 @@ void Daemon::tcmalloc_gc_thread() {
// however, it is hard to set limit on cache of tcmalloc and doris
// use mmap in vectorized mode.
// Limit cache capactiy is enough.
- int64_t min_free_bytes = alloc_bytes - physical_limit_bytes * 9 / 10;
- to_free_bytes = std::max(to_free_bytes, min_free_bytes);
- to_free_bytes = std::max(to_free_bytes, tc_cached_bytes * 30 / 100);
- // We assure that we have at least 500M bytes in cache.
- to_free_bytes = std::min(to_free_bytes, tc_cached_bytes - 500 * 1024 * 1024);
- expected_aggressive_decommit = (memory_pressure_no_cache > pressure_limit);
+ if (rss_pressure > pressure_limit) {
+ int64_t min_free_bytes = alloc_bytes - physical_limit_bytes * 9 / 10;
+ to_free_bytes = std::max(to_free_bytes, min_free_bytes);
+ to_free_bytes = std::max(to_free_bytes, tc_cached_bytes * 30 / 100);
+ // We assure that we have at least 500M bytes in cache.
+ to_free_bytes = std::min(to_free_bytes, tc_cached_bytes - 500 * 1024 * 1024);
+ expected_aggressive_decommit = 1;
+ }
last_ms = kMaxLastMs;
} else if (memory_pressure > (pressure_limit - 10)) {
// In most cases, adjusting release rate is enough, if memory are consumed quickly
@@ -143,7 +146,7 @@ void Daemon::tcmalloc_gc_thread() {
int release_rate_index = memory_pressure / 10;
double release_rate = 1.0;
- if (release_rate_index >= sizeof(release_rates)) {
+ if (release_rate_index >= sizeof(release_rates) / sizeof(release_rates[0])) {
release_rate = 2000.0;
} else {
release_rate = release_rates[release_rate_index];
@@ -158,7 +161,7 @@ void Daemon::tcmalloc_gc_thread() {
last_memory_pressure = memory_pressure;
// We release at least 2% bytes once, frequent releasing hurts performance.
- if (to_free_bytes > physical_limit_bytes * 2 / 100) {
+ if (to_free_bytes > (physical_limit_bytes * 2 / 100)) {
last_ms += kIntervalMs;
if (last_ms >= kMaxLastMs) {
LOG(INFO) << "generic.current_allocated_bytes " << tc_used_bytes
@@ -170,7 +173,7 @@ void Daemon::tcmalloc_gc_thread() {
<< current_aggressive_decommit;
MallocExtension::instance()->ReleaseToSystem(to_free_bytes);
last_ms = 0;
- }
+ }
} else {
last_ms = 0;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org