You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2023/12/26 08:22:59 UTC
(doris) branch master updated: [enhencement](config) change default memtable size & loadStreamPerNode & default load parallelism (#28977)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1964a77d6c7 [enhencement](config) change default memtable size & loadStreamPerNode & default load parallelism (#28977)
1964a77d6c7 is described below
commit 1964a77d6c7694dc956ffe0d4a73b61205f3ccaf
Author: zhengyu <fr...@gmail.com>
AuthorDate: Tue Dec 26 16:22:52 2023 +0800
[enhencement](config) change default memtable size & loadStreamPerNode & default load parallelism (#28977)
We change memtable size from 200MB to 100MB to achieve smoother flush
performance. We change loadStreamPerNode from 20 to 60 to avoid stream
rpc to be the bottleneck when enable memtable_on_sink_node. We change
default s3&broker load parallelsim to make the most of CPUs on moderm
multi-core systems.
Signed-off-by: freemandealer <fr...@gmail.com>
---
be/src/common/config.cpp | 2 +-
be/src/common/config.h | 2 +-
docs/en/docs/admin-manual/config/fe-config.md | 2 +-
docs/zh-CN/docs/admin-manual/config/fe-config.md | 2 +-
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1d8178ceec2..7dc1ec680a1 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -592,7 +592,7 @@ DEFINE_Int32(memory_max_alignment, "16");
// memtable insert memory tracker will multiply input block size with this ratio
DEFINE_mDouble(memtable_insert_memory_ratio, "1.4");
// max write buffer size before flush, default 200MB
-DEFINE_mInt64(write_buffer_size, "209715200");
+DEFINE_mInt64(write_buffer_size, "104857600");
// max buffer size used in memtable for the aggregated table, default 400MB
DEFINE_mInt64(write_buffer_size_for_agg, "419430400");
// max parallel flush task per memtable writer
diff --git a/be/src/common/config.h b/be/src/common/config.h
index affe47851fb..bb341002902 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -642,7 +642,7 @@ DECLARE_Int32(memory_max_alignment);
// memtable insert memory tracker will multiply input block size with this ratio
DECLARE_mDouble(memtable_insert_memory_ratio);
-// max write buffer size before flush, default 200MB
+// max write buffer size before flush, default 100MB
DECLARE_mInt64(write_buffer_size);
// max buffer size used in memtable for the aggregated table, default 400MB
DECLARE_mInt64(write_buffer_size_for_agg);
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index 3721a364d50..ae3c0645be4 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -1153,7 +1153,7 @@ Max bytes a broker scanner can process in one broker load job. Commonly, each Ba
#### `default_load_parallelism`
-Default: 1
+Default: 8
IsMutable：true
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index 356677ae04e..580cfe7fee1 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -1154,7 +1154,7 @@ broker scanner 程序可以在一个 broker 加载作业中处理的最大字节
#### `default_load_parallelism`
-默认值：1
+默认值：8
是否可以动态配置：true
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index c038a9f1780..61a135ed963 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -144,7 +144,7 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true, description = {"broker load 时，单个节点上 load 执行计划的默认并行度",
"The default parallelism of the load execution plan on a single node when the broker load is submitted"})
- public static int default_load_parallelism = 1;
+ public static int default_load_parallelism = 8;
@ConfField(mutable = true, masterOnly = true, description = {
"已完成或取消的导入作业信息的 label 会在这个时间后被删除。被删除的 label 可以被重用。",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 332e0f5bfaa..e33a6193674 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1325,7 +1325,7 @@ public class SessionVariable implements Serializable, Writable {
public boolean enableMemtableOnSinkNode = true;
@VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
- public int loadStreamPerNode = 20;
+ public int loadStreamPerNode = 60;
@VariableMgr.VarAttr(name = GROUP_COMMIT)
public String groupCommit = "off_mode";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org