You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/01/10 12:48:20 UTC
[incubator-doris] branch master updated: [Config] Add publish
timeout param when exec insert (#5170)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 05631cf [Config] Add publish timeout param when exec insert (#5170)
05631cf is described below
commit 05631cfa4f5fcec78d80ee015f981bb682976426
Author: Lijia Liu <li...@yeah.net>
AuthorDate: Sun Jan 10 20:48:10 2021 +0800
[Config] Add publish timeout param when exec insert (#5170)
Add new session variable to control the timeout of publish task of insert operation.
---
docs/en/administrator-guide/variables.md | 8 +++++++-
docs/zh-CN/administrator-guide/variables.md | 7 ++++++-
.../java/org/apache/doris/qe/ConnectProcessor.java | 4 ++++
.../java/org/apache/doris/qe/MasterOpExecutor.java | 1 +
.../java/org/apache/doris/qe/SessionVariable.java | 23 ++++++++++++++++++++++
.../java/org/apache/doris/qe/StmtExecutor.java | 3 +--
gensrc/thrift/FrontendService.thrift | 1 +
7 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md
index 3328115..468df82 100644
--- a/docs/en/administrator-guide/variables.md
+++ b/docs/en/administrator-guide/variables.md
@@ -74,6 +74,7 @@ Variables that support both session-level and global-level setting include:
* `parallel_fragment_exec_instance_num`
* `parallel_exchange_instance_num`
* `allow_partition_column_nullable`
+* `insert_visible_timeout_ms`
Variables that support only global-level setting include:
@@ -364,4 +365,9 @@ Note that the comment must start with /*+ and can only follow the SELECT.
* `allow_partition_column_nullable`
- Whether to allow the partition column to be NULL when creating the table. The default is true, which means NULL is allowed. false means the partition column must be defined as NOT NULL.
\ No newline at end of file
+ Whether to allow the partition column to be NULL when creating the table. The default is true, which means NULL is allowed. false means the partition column must be defined as NOT NULL.
+
+* `insert_visible_timeout_ms`
+
+ When execute insert statement, doris will wait for the transaction to commit and visible after the import is completed.
+ This parameter controls the timeout of waiting for transaction to be visible. The default value is 10000, and the minimum value is 1000.
\ No newline at end of file
diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md
index 1b71c06..00df741 100644
--- a/docs/zh-CN/administrator-guide/variables.md
+++ b/docs/zh-CN/administrator-guide/variables.md
@@ -74,6 +74,7 @@ SET GLOBAL exec_mem_limit = 137438953472
* `parallel_fragment_exec_instance_num`
* `parallel_exchange_instance_num`
* `allow_partition_column_nullable`
+* `insert_visible_timeout_ms`
只支持全局生效的变量包括：
@@ -363,4 +364,8 @@ SELECT /*+ SET_VAR(query_timeout = 1) */ sleep(3);
* `allow_partition_column_nullable`
- 建表时是否允许分区列为NULL。默认为true，表示允许为NULL。false 表示分区列必须被定义为NOT NULL
\ No newline at end of file
+ 建表时是否允许分区列为NULL。默认为true，表示允许为NULL。false 表示分区列必须被定义为NOT NULL
+
+* `insert_visible_timeout_ms`
+
+ 在执行insert语句时，导入动作(查询和插入)完成后，还需要等待事务提交，使数据可见。此参数控制等待数据可见的超时时间，默认为10000，最小为1000。
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index e32cbee..679e184 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -415,6 +415,10 @@ public class ConnectProcessor {
ctx.setCurrentUserIdentity(currentUserIdentity);
}
+ if (request.isSetInsertVisibleTimeoutMs()) {
+ ctx.getSessionVariable().setInsertVisibleTimeoutMs(request.getInsertVisibleTimeoutMs());
+ }
+
if (request.isSetQueryOptions()) {
TQueryOptions queryOptions = request.getQueryOptions();
if (queryOptions.isSetMemLimit()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
index e222a2e..fa62728 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
@@ -93,6 +93,7 @@ public class MasterOpExecutor {
params.setStmtId(ctx.getStmtId());
params.setEnableStrictMode(ctx.getSessionVariable().getEnableInsertStrict());
params.setCurrentUserIdent(ctx.getCurrentUserIdentity().toThrift());
+ params.setInsertVisibleTimeoutMs(ctx.getSessionVariable().getInsertVisibleTimeoutMs());
TQueryOptions queryOptions = new TQueryOptions();
queryOptions.setMemLimit(ctx.getSessionVariable().getMaxExecMemByte());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1a96c13..885c543 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -112,6 +112,14 @@ public class SessionVariable implements Serializable, Writable {
// when true, the partition column must be set to NOT NULL.
public static final String ALLOW_PARTITION_COLUMN_NULLABLE = "allow_partition_column_nullable";
+ // max ms to wait transaction publish finish when exec insert stmt.
+ public static final String INSERT_VISIBLE_TIMEOUT_MS = "insert_visible_timeout_ms";
+ public static final long DEFAULT_INSERT_VISIBLE_TIMEOUT_MS = 10_000;
+ public static final long MIN_INSERT_VISIBLE_TIMEOUT_MS = 1000; // If user set a very small value, use this value instead.
+
+ @VariableMgr.VarAttr(name = INSERT_VISIBLE_TIMEOUT_MS)
+ private long insertVisibleTimeoutMs = DEFAULT_INSERT_VISIBLE_TIMEOUT_MS;
+
// max memory used on every backend.
@VariableMgr.VarAttr(name = EXEC_MEM_LIMIT)
public long maxExecMemByte = 2147483648L;
@@ -544,6 +552,21 @@ public class SessionVariable implements Serializable, Writable {
public boolean isAllowPartitionColumnNullable() { return allowPartitionColumnNullable; }
+ public long getInsertVisibleTimeoutMs() {
+ if (insertVisibleTimeoutMs < MIN_INSERT_VISIBLE_TIMEOUT_MS) {
+ return MIN_INSERT_VISIBLE_TIMEOUT_MS;
+ } else {
+ return insertVisibleTimeoutMs;
+ }
+ }
+
+ public void setInsertVisibleTimeoutMs(long insertVisibleTimeoutMs) {
+ if (insertVisibleTimeoutMs < MIN_INSERT_VISIBLE_TIMEOUT_MS) {
+ this.insertVisibleTimeoutMs = MIN_INSERT_VISIBLE_TIMEOUT_MS;
+ } else {
+ this.insertVisibleTimeoutMs = insertVisibleTimeoutMs;
+ }
+ }
// Serialize to thrift object
// used for rest api
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 6e33853..d2b51ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -897,11 +897,10 @@ public class StmtExecutor {
context.getState().setOk();
return;
}
-
if (Catalog.getCurrentGlobalTransactionMgr().commitAndPublishTransaction(
insertStmt.getDbObj(), insertStmt.getTransactionId(),
TabletCommitInfo.fromThrift(coord.getCommitInfos()),
- 10000)) {
+ context.getSessionVariable().getInsertVisibleTimeoutMs())) {
txnStatus = TransactionStatus.VISIBLE;
MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
} else {
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index b10fbe7..df3ddbe 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -443,6 +443,7 @@ struct TMasterOpRequest {
15: optional i32 stmtIdx // the idx of the sql in multi statements
16: optional PaloInternalService.TQueryOptions query_options
17: optional Types.TUniqueId query_id // when this is a query, we translate this query id to master
+ 18: optional i64 insert_visible_timeout_ms
}
struct TColumnDefinition {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org