You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2023/01/30 02:29:50 UTC
[doris] branch branch-1.2-lts updated: [enhancement](memory) reduce memory usage for failed broker loads (#15895)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new afafd20359 [enhancement](memory) reduce memory usage for failed broker loads (#15895)
afafd20359 is described below
commit afafd2035951a99d450ce60366054af323512ec1
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Mon Jan 30 10:22:31 2023 +0800
[enhancement](memory) reduce memory usage for failed broker loads (#15895)
* [enhancement](memory) reduce memory usage for failed broker loads
t # Conflicts:
---
docs/en/docs/admin-manual/config/fe-config.md | 9 +++++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 ++++++++
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 6 ++++++
.../main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 5 ++++-
.../src/main/java/org/apache/doris/load/loadv2/LoadJob.java | 3 ++-
5 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index dd974a89b0..20cdc89a9c 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2378,4 +2378,13 @@ MasterOnly：false
The default dir to put jdbc drivers.
+#### `max_error_tablet_of_broker_load`
+
+Default: 3;
+
+IsMutable：true
+
+MasterOnly：true
+
+Maximum number of error tablet showed in broker load.
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index e80abac136..2888279e0a 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2440,4 +2440,12 @@ hive partition 的最大缓存数量。
用于存放默认的 jdbc drivers
+#### `max_error_tablet_of_broker_load`
+默认值：3;
+
+是否可以动态配置: true
+
+是否为 Master FE 节点独有的配置项：true
+
+broker load job 保存的失败tablet 信息的最大数量
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 487cf430d8..89eb179acd 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1960,5 +1960,11 @@ public class Config extends ConfigBase {
*/
@ConfField(masterOnly = true)
public static int hms_events_polling_interval_ms = 10000;
+
+ /**
+ * Maximum number of error tablets showed in broker load
+ */
+ @ConfField(masterOnly = true, mutable = true)
+ public static int max_error_tablet_of_broker_load = 3;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index c8c0e15e0c..c396297afd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DataQualityException;
import org.apache.doris.common.DuplicatedRequestException;
import org.apache.doris.common.LabelAlreadyUsedException;
@@ -59,6 +60,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.RejectedExecutionException;
+import java.util.stream.Collectors;
/**
* There are 3 steps in BrokerLoadJob: BrokerPendingTask, LoadLoadingTask, CommitAndPublishTxn.
@@ -344,7 +346,8 @@ public class BrokerLoadJob extends BulkLoadJob {
loadingStatus.setTrackingUrl(attachment.getTrackingUrl());
}
commitInfos.addAll(attachment.getCommitInfoList());
- errorTabletInfos.addAll(attachment.getErrorTabletInfos());
+ errorTabletInfos.addAll(attachment.getErrorTabletInfos().stream().limit(Config.max_error_tablet_of_broker_load)
+ .collect(Collectors.toList()));
progress = (int) ((double) finishedTaskIds.size() / idToTasks.size() * 100);
if (progress == 100) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
index 5e9013fa34..fc74415940 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -797,7 +797,8 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
public String errorTabletsToJson() {
Map<Long, String> map = Maps.newHashMap();
- errorTabletInfos.stream().limit(3).forEach(p -> map.put(p.getTabletId(), p.getMsg()));
+ errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load)
+ .forEach(p -> map.put(p.getTabletId(), p.getMsg()));
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
return gson.toJson(map);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org