You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/06/19 09:09:35 UTC
[incubator-doris] branch master updated: [Config] Support
max_stream_load_timeout_second config in fe (#3902)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d40218 [Config] Support max_stream_load_timeout_second config in fe (#3902)
5d40218 is described below
commit 5d40218ae6c1bbee538f2c1c0329477ea007078e
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Fri Jun 19 17:09:27 2020 +0800
[Config] Support max_stream_load_timeout_second config in fe (#3902)
This configuration is specifically used to limit timeout setting for stream load.
It is to prevent that failed stream load transactions cannot be canceled within
a short time because of the user's large timeout setting.
---
docs/en/administrator-guide/config/fe_config.md | 4 ++++
docs/zh-CN/administrator-guide/config/fe_config.md | 4 ++++
.../main/java/org/apache/doris/common/Config.java | 8 +++++++-
.../doris/transaction/GlobalTransactionMgr.java | 22 ++++++++++++++++------
4 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md
index 888a075..411076f 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -413,6 +413,10 @@ Generally it is not recommended to increase this configuration value. An excessi
### `max_small_file_size_bytes`
+### `max_stream_load_timeout_second`
+
+This configuration is specifically used to limit timeout setting for stream load. It is to prevent that failed stream load transactions cannot be canceled within a short time because of the user's large timeout setting.
+
### `max_tolerable_backend_down_num`
### `max_unfinished_load_job`
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md
index e68fb6b..6103a05 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -411,6 +411,10 @@ current running txns on db xxx is xx, larger than limit xx
### `max_small_file_size_bytes`
+### `max_stream_load_timeout_second`
+
+该配置是专门用来限制stream load的超时时间配置，防止失败的stream load事务因为用户的超长时间设置无法在短时间内被取消掉。
+
### `max_tolerable_backend_down_num`
### `max_unfinished_load_job`
diff --git a/fe/src/main/java/org/apache/doris/common/Config.java b/fe/src/main/java/org/apache/doris/common/Config.java
index 362dd48..ef38967 100644
--- a/fe/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/src/main/java/org/apache/doris/common/Config.java
@@ -474,12 +474,18 @@ public class Config extends ConfigBase {
public static int stream_load_default_timeout_second = 600; // 600s
/*
- * Max load timeout applicable to all type of load
+ * Max load timeout applicable to all type of load except for stream load
*/
@ConfField(mutable = true, masterOnly = true)
public static int max_load_timeout_second = 259200; // 3days
/*
+ * Max stream load and streaming mini load timeout
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static int max_stream_load_timeout_second = 259200; // 3days
+
+ /*
* Min stream load timeout applicable to all type of load
*/
@ConfField(mutable = true, masterOnly = true)
diff --git a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
index 41f82bc..16c813a 100644
--- a/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
+++ b/fe/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
@@ -116,18 +116,28 @@ public class GlobalTransactionMgr implements Writable {
if (Config.disable_load_job) {
throw new AnalysisException("disable_load_job is set to true, all load jobs are prevented");
}
-
- if (timeoutSecond > Config.max_load_timeout_second ||
- timeoutSecond < Config.min_load_timeout_second) {
- throw new AnalysisException("Invalid timeout. Timeout should between "
- + Config.min_load_timeout_second + " and " + Config.max_load_timeout_second
- + " seconds");
+
+ switch (sourceType) {
+ case BACKEND_STREAMING:
+ checkValidTimeoutSecond(timeoutSecond, Config.max_stream_load_timeout_second, Config.min_load_timeout_second);
+ break;
+ default:
+ checkValidTimeoutSecond(timeoutSecond, Config.max_load_timeout_second, Config.min_load_timeout_second);
}
DatabaseTransactionMgr dbTransactionMgr = getDatabaseTransactionMgr(dbId);
return dbTransactionMgr.beginTransaction(tableIdList, label, requestId, coordinator, sourceType, listenerId, timeoutSecond);
}
+ private void checkValidTimeoutSecond(long timeoutSecond, int maxLoadTimeoutSecond, int minLoadTimeOutSecond) throws AnalysisException {
+ if (timeoutSecond > maxLoadTimeoutSecond ||
+ timeoutSecond < minLoadTimeOutSecond) {
+ throw new AnalysisException("Invalid timeout. Timeout should between "
+ + minLoadTimeOutSecond + " and " + maxLoadTimeoutSecond
+ + " seconds");
+ }
+ }
+
public TransactionStatus getLabelState(long dbId, String label) {
try {
DatabaseTransactionMgr dbTransactionMgr = getDatabaseTransactionMgr(dbId);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org