You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2023/01/20 15:55:51 UTC
[GitHub] [doris] morningman commented on a diff in pull request #16084: [improvement](memory) Optimize the log of process memory insufficient and support regular GC cache
morningman commented on code in PR #16084:
URL: https://github.com/apache/doris/pull/16084#discussion_r1082727190
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug", false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >= doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full gc", false);
Review Comment:
Looks like it will print too many logs when memory is insuffient?
Because the interval is only 500ms?
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug", false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >= doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full gc", false);
+ interval_milliseconds = std::min(100, config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor gc", false);
+ interval_milliseconds = std::min(200, config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_minor_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else {
interval_milliseconds = config::memory_maintenance_sleep_time_ms;
+ if (doris::config::memory_debug) {
+ doris::MemTrackerLimiter::print_log_process_usage("memory debug", false);
+ } else {
+ LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
+ }
+ cache_gc_interval_ms -= interval_milliseconds;
+ if (cache_gc_interval_ms < 0) {
+ cache_gc_freed_mem = 0;
+ doris::MemInfo::process_cache_gc(cache_gc_freed_mem);
+ LOG(INFO) << fmt::format("Process regular GC Cache, Free Memory {} Bytes",
Review Comment:
ditto
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug", false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >= doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full gc", false);
+ interval_milliseconds = std::min(100, config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor gc", false);
Review Comment:
ditto
##########
be/src/common/daemon.cpp:
##########
@@ -203,29 +205,42 @@ void Daemon::memory_maintenance_thread() {
doris::MemInfo::refresh_allocator_mem();
#endif
doris::MemInfo::refresh_proc_mem_no_allocator_cache();
- LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
// Refresh mem tracker each type metrics.
doris::MemTrackerLimiter::refresh_global_counter();
- if (doris::config::memory_debug) {
- doris::MemTrackerLimiter::print_log_process_usage("memory_debug", false);
- }
doris::MemTrackerLimiter::enable_print_log_process_usage();
// If system available memory is not enough, or the process memory exceeds the limit, reduce refresh interval.
if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_low_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >= doris::MemInfo::mem_limit()) {
- interval_milliseconds = 100;
+ doris::MemTrackerLimiter::print_log_process_usage("process full gc", false);
+ interval_milliseconds = std::min(100, config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_full_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else if (doris::MemInfo::sys_mem_available() <
doris::MemInfo::sys_mem_available_warning_water_mark() ||
doris::MemInfo::proc_mem_no_allocator_cache() >=
doris::MemInfo::soft_mem_limit()) {
- interval_milliseconds = 200;
+ doris::MemTrackerLimiter::print_log_process_usage("process minor gc", false);
+ interval_milliseconds = std::min(200, config::memory_maintenance_sleep_time_ms);
doris::MemInfo::process_minor_gc();
+ cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
} else {
interval_milliseconds = config::memory_maintenance_sleep_time_ms;
+ if (doris::config::memory_debug) {
+ doris::MemTrackerLimiter::print_log_process_usage("memory debug", false);
+ } else {
+ LOG_EVERY_N(INFO, 10) << MemTrackerLimiter::process_mem_log_str();
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org