You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/28 11:46:18 UTC
[doris] 01/03: [fix](memory) mmap threshold can be modified in conf, Increase to 128M
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0-alpha
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a967cdc1e4d73b038581b4d9cf03ebd629128c5b
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Fri Apr 28 18:17:22 2023 +0800
[fix](memory) mmap threshold can be modified in conf, Increase to 128M
---
be/src/common/config.h | 14 +++++++++++
be/src/vec/common/allocator.h | 54 +++++++++++++++----------------------------
2 files changed, 32 insertions(+), 36 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 976dfd3149..65ce764be1 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -67,6 +67,20 @@ CONF_String(mem_limit, "auto");
// Soft memory limit as a fraction of hard memory limit.
CONF_Double(soft_mem_limit_frac, "0.9");
+// Many modern allocators (for example, tcmalloc) do not do a mremap for
+// realloc, even in case of large enough chunks of memory. Although this allows
+// you to increase performance and reduce memory consumption during realloc.
+// To fix this, we do mremap manually if the chunk of memory is large enough.
+//
+// The threshold (128 MB, 128 * (1ULL << 20)) is chosen quite large, since changing the address
+// space is very slow, especially in the case of a large number of threads. We
+// expect that the set of operations mmap/something to do/mremap can only be
+// performed about 1000 times per second.
+//
+// P.S. This is also required, because tcmalloc can not allocate a chunk of
+// memory greater than 16 GB.
+CONF_mInt64(mmap_threshold, "134217728"); // bytes
+
// When hash table capacity is greater than 2^double_grow_degree(default 2G), grow when 75% of the capacity is satisfied.
// Increase can reduce the number of hash table resize, but may waste more memory.
CONF_mInt32(hash_table_double_grow_degree, "31");
diff --git a/be/src/vec/common/allocator.h b/be/src/vec/common/allocator.h
index 7ae7ff7082..0be19e88d8 100644
--- a/be/src/vec/common/allocator.h
+++ b/be/src/vec/common/allocator.h
@@ -61,21 +61,6 @@
#define MAP_ANONYMOUS MAP_ANON
#endif
-#ifdef NDEBUG
-/**
- * Many modern allocators (for example, tcmalloc) do not do a mremap for
- * realloc, even in case of large enough chunks of memory. Although this allows
- * you to increase performance and reduce memory consumption during realloc.
- * To fix this, we do mremap manually if the chunk of memory is large enough.
- * The threshold (64 MB) is chosen quite large, since changing the address
- * space is very slow, especially in the case of a large number of threads. We
- * expect that the set of operations mmap/something to do/mremap can only be
- * performed about 1000 times per second.
- *
- * P.S. This is also required, because tcmalloc can not allocate a chunk of
- * memory greater than 16 GB.
- */
-static constexpr size_t MMAP_THRESHOLD = 64 * (1ULL << 20);
/**
* Memory allocation between 4KB and 64MB will be through ChunkAllocator,
* those less than 4KB will be through malloc (for example, tcmalloc),
@@ -86,15 +71,12 @@ static constexpr size_t MMAP_THRESHOLD = 64 * (1ULL << 20);
* by more detailed test later.
*/
static constexpr size_t CHUNK_THRESHOLD = 4096;
-#else
/**
* In debug build, use small mmap threshold to reproduce more memory
* stomping bugs. Along with ASLR it will hopefully detect more issues than
* ASan. The program may fail due to the limit on number of memory mappings.
*/
-static constexpr size_t MMAP_THRESHOLD = 4096;
-static constexpr size_t CHUNK_THRESHOLD = 1024;
-#endif
+static constexpr size_t MMAP_THRESHOLD_DEBUG = 4096; // delete immediately
static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
@@ -127,7 +109,11 @@ public:
memory_check(size);
void* buf;
- if (size >= MMAP_THRESHOLD) {
+#ifdef NDEBUG
+ if (size >= doris::config::mmap_threshold) {
+#else
+ if (size >= MMAP_THRESHOLD_DEBUG) {
+#endif
if (alignment > MMAP_MIN_ALIGNMENT)
throw doris::Exception(
doris::ErrorCode::INVALID_ARGUMENT,
@@ -135,7 +121,7 @@ public:
alignment, size);
consume_memory(size);
- buf = mmap(get_mmap_hint(), size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
+ buf = mmap(nullptr, size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
if (MAP_FAILED == buf) {
release_memory(size);
throw_bad_alloc(fmt::format("Allocator: Cannot mmap {}.", size));
@@ -176,7 +162,11 @@ public:
/// Free memory range.
void free(void* buf, size_t size) {
- if (size >= MMAP_THRESHOLD) {
+#ifdef NDEBUG
+ if (size >= doris::config::mmap_threshold) {
+#else
+ if (size >= MMAP_THRESHOLD_DEBUG) {
+#endif
if (0 != munmap(buf, size)) {
throw_bad_alloc(fmt::format("Allocator: Cannot munmap {}.", size));
} else {
@@ -213,7 +203,12 @@ public:
if constexpr (clear_memory)
if (new_size > old_size)
memset(reinterpret_cast<char*>(buf) + old_size, 0, new_size - old_size);
- } else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD) {
+#ifdef NDEBUG
+ } else if (old_size >= doris::config::mmap_threshold &&
+ new_size >= doris::config::mmap_threshold) {
+#else
+ } else if (old_size >= MMAP_THRESHOLD_DEBUG && new_size >= MMAP_THRESHOLD_DEBUG) {
+#endif
memory_check(new_size);
/// Resize mmap'd memory region.
consume_memory(new_size - old_size);
@@ -264,19 +259,6 @@ protected:
| (mmap_populate ? MAP_POPULATE : 0)
#endif
;
-
-private:
-#ifndef NDEBUG
- /// In debug builds, request mmap() at random addresses (a kind of ASLR), to
- /// reproduce more memory stomping bugs. Note that Linux doesn't do it by
- /// default. This may lead to worse TLB performance.
- void* get_mmap_hint() {
- // return reinterpret_cast<void *>(std::uniform_int_distribution<intptr_t>(0x100000000000UL, 0x700000000000UL)(thread_local_rng));
- return nullptr;
- }
-#else
- void* get_mmap_hint() { return nullptr; }
-#endif
};
/** Allocator with optimization to place small memory ranges in automatic memory.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org