You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/10/09 08:20:06 UTC
[doris] branch master updated: [chore](be config) remove config use_mmap_allocate_chunk #13196
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc2d33298b [chore](be config) remove config use_mmap_allocate_chunk #13196
dc2d33298b is described below
commit dc2d33298b225489033f547d1bb6d15c6daf0e1b
Author: yiguolei <67...@qq.com>
AuthorDate: Sun Oct 9 16:19:59 2022 +0800
[chore](be config) remove config use_mmap_allocate_chunk #13196
This config is never used online and there exist bugs if enable this config. So that I remove this config and related tests.
Co-authored-by: yiguolei <yi...@gmail.com>
---
be/src/common/config.h | 8 ------
be/src/runtime/memory/system_allocator.cpp | 33 ++----------------------
be/test/runtime/memory/chunk_allocator_test.cpp | 1 -
be/test/runtime/memory/system_allocator_test.cpp | 5 +---
docs/en/docs/admin-manual/config/be-config.md | 6 -----
docs/zh-CN/docs/admin-manual/config/be-config.md | 6 -----
6 files changed, 3 insertions(+), 56 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 80e821c536..e0baed3da4 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -437,14 +437,6 @@ CONF_Int32(max_free_io_buffers, "128");
// including MemPool, ChunkAllocator, BufferPool, DiskIO free buffer.
CONF_Bool(disable_mem_pools, "false");
-// Whether to allocate chunk using mmap. If you enable this, you'd better to
-// increase vm.max_map_count's value whose default value is 65530.
-// you can do it as root via "sysctl -w vm.max_map_count=262144" or
-// "echo 262144 > /proc/sys/vm/max_map_count"
-// NOTE: When this is set to true, you must set chunk_reserved_bytes_limit
-// to a relative large number or the performance is very very bad.
-CONF_Bool(use_mmap_allocate_chunk, "false");
-
// The reserved bytes limit of Chunk Allocator, usually set as a percentage of mem_limit.
// defaults to bytes if no unit is given, the number of bytes must be a multiple of 2.
// must larger than 0. and if larger than physical memory size, it will be set to physical memory size.
diff --git a/be/src/runtime/memory/system_allocator.cpp b/be/src/runtime/memory/system_allocator.cpp
index c05c28861b..fe268ec8d3 100644
--- a/be/src/runtime/memory/system_allocator.cpp
+++ b/be/src/runtime/memory/system_allocator.cpp
@@ -30,26 +30,11 @@ namespace doris {
#define PAGE_SIZE (4 * 1024) // 4K
uint8_t* SystemAllocator::allocate(size_t length) {
- if (config::use_mmap_allocate_chunk) {
- return allocate_via_mmap(length);
- } else {
- return allocate_via_malloc(length);
- }
+ return allocate_via_malloc(length);
}
void SystemAllocator::free(uint8_t* ptr, size_t length) {
- if (config::use_mmap_allocate_chunk) {
- auto res = munmap(ptr, length);
- if (res != 0) {
- char buf[64];
- LOG(ERROR) << "fail to free memory via munmap, errno=" << errno
- << ", errmsg=" << strerror_r(errno, buf, 64);
- } else {
- RELEASE_THREAD_MEM_TRACKER(length);
- }
- } else {
- ::free(ptr);
- }
+ ::free(ptr);
}
uint8_t* SystemAllocator::allocate_via_malloc(size_t length) {
@@ -65,18 +50,4 @@ uint8_t* SystemAllocator::allocate_via_malloc(size_t length) {
return (uint8_t*)ptr;
}
-uint8_t* SystemAllocator::allocate_via_mmap(size_t length) {
- CONSUME_THREAD_MEM_TRACKER(length);
- auto ptr = (uint8_t*)mmap(nullptr, length, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
- -1, 0);
- if (ptr == MAP_FAILED) {
- char buf[64];
- LOG(ERROR) << "fail to allocate memory via mmap, errno=" << errno
- << ", errmsg=" << strerror_r(errno, buf, 64);
- RELEASE_THREAD_MEM_TRACKER(length);
- return nullptr;
- }
- return ptr;
-}
-
} // namespace doris
diff --git a/be/test/runtime/memory/chunk_allocator_test.cpp b/be/test/runtime/memory/chunk_allocator_test.cpp
index aaf3d91b13..b3854724eb 100644
--- a/be/test/runtime/memory/chunk_allocator_test.cpp
+++ b/be/test/runtime/memory/chunk_allocator_test.cpp
@@ -28,7 +28,6 @@
namespace doris {
TEST(ChunkAllocatorTest, Normal) {
- config::use_mmap_allocate_chunk = true;
for (size_t size = 4096; size <= 1024 * 1024; size <<= 1) {
Chunk chunk;
EXPECT_TRUE(ChunkAllocator::instance()->allocate(size, &chunk).ok());
diff --git a/be/test/runtime/memory/system_allocator_test.cpp b/be/test/runtime/memory/system_allocator_test.cpp
index 3d7a471aa5..6a155ce052 100644
--- a/be/test/runtime/memory/system_allocator_test.cpp
+++ b/be/test/runtime/memory/system_allocator_test.cpp
@@ -23,9 +23,7 @@
namespace doris {
-template <bool use_mmap>
void test_normal() {
- config::use_mmap_allocate_chunk = use_mmap;
{
auto ptr = SystemAllocator::allocate(4096);
EXPECT_NE(nullptr, ptr);
@@ -41,8 +39,7 @@ void test_normal() {
}
TEST(SystemAllocatorTest, TestNormal) {
- test_normal<true>();
- test_normal<false>();
+ test_normal();
}
} // namespace doris
diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md
index cdabd0fa67..d8fa6a11fa 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -1454,12 +1454,6 @@ Default: 1
Maximum number of threads for uploading files
-### `use_mmap_allocate_chunk`
-
-Default: false
-
-Whether to use mmap to allocate blocks. If you enable this feature, it is best to increase the value of vm.max_map_count, its default value is 65530. You can use "sysctl -w vm.max_map_count=262144" or "echo 262144> /proc/sys/vm/" to operate max_map_count as root. When this setting is true, you must set chunk_reserved_bytes_limit to a relatively low Big number, otherwise the performance is very very bad
-
### `user_function_dir`
${DORIS_HOME}/lib/udf
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md
index 13f62274be..758b93bc91 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -1477,12 +1477,6 @@ txn_lock 分片大小，取值为2^n，n=0,1,2,3,4， 这是一项增强功能
上传文件最大线程数
-### `use_mmap_allocate_chunk`
-
-默认值：false
-
-是否使用 mmap 分配块。 如果启用此功能，最好增加 vm.max_map_count 的值，其默认值为 65530。您可以通过“sysctl -w vm.max_map_count=262144”或“echo 262144 > /proc/sys/vm/”以 root 身份进行操作 max_map_count" ，当这个设置为true时，你必须将chunk_reserved_bytes_limit设置为一个相对较大的数字，否则性能非常非常糟糕。
-
### `user_function_dir`
默认值：${DORIS_HOME}/lib/udf
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org