You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/10 06:55:56 UTC
(doris) 03/11: [Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1b3a11a02b79a33a8d66a0a48bc27fb26ddd253b
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Mon Apr 1 22:53:21 2024 +0800
[Enhancement](merge-on-write) Support dynamic delete bitmap cache (#32991)
* The default delete bitmap cache is set to 100MB, which can be insufficient and cause performance issues when the amount of user data is large. To mitigate the problem of an inadequate cache, we will take the larger of 5% of the total memory and 100MB as the delete bitmap cache size.
---
be/src/common/config.cpp | 5 +++++
be/src/common/config.h | 1 +
be/src/olap/tablet_meta.cpp | 15 ++++++++++++++-
3 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index f3bde7dd36b..17a5587f073 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -797,6 +797,11 @@ DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800");
// Global bitmap cache capacity for aggregation cache, size in bytes
DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
+// The default delete bitmap cache is set to 100MB,
+// which can be insufficient and cause performance issues when the amount of user data is large.
+// To mitigate the problem of an inadequate cache,
+// we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size.
+DEFINE_String(delete_bitmap_dynamic_agg_cache_limit, "0.5%");
DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
// reference https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#broker-version-compatibility
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a0845d043de..6076ce79433 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -851,6 +851,7 @@ DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec);
// Global bitmap cache capacity for aggregation cache, size in bytes
DECLARE_Int64(delete_bitmap_agg_cache_capacity);
+DECLARE_String(delete_bitmap_dynamic_agg_cache_limit);
DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
// A common object cache depends on an Sharded LRU Cache.
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index d8235191b93..f8a3bb9e97a 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -38,6 +38,8 @@
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
#include "util/debug_points.h"
+#include "util/mem_info.h"
+#include "util/parse_util.h"
#include "util/string_util.h"
#include "util/time.h"
#include "util/uid_util.h"
@@ -924,7 +926,18 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) {
}
DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {
- _agg_cache.reset(new AggCache(config::delete_bitmap_agg_cache_capacity));
+ // The default delete bitmap cache is set to 100MB,
+ // which can be insufficient and cause performance issues when the amount of user data is large.
+ // To mitigate the problem of an inadequate cache,
+ // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size.
+ bool is_percent = false;
+ int64_t delete_bitmap_agg_cache_cache_limit =
+ ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit,
+ MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent);
+ _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit >
+ config::delete_bitmap_agg_cache_capacity
+ ? delete_bitmap_agg_cache_cache_limit
+ : config::delete_bitmap_agg_cache_capacity));
}
DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org