You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/08/03 05:25:58 UTC
[doris] branch branch-2.0 updated (280f612548 -> bd443bbac7)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 280f612548 [fix][branch-2.0](memory) fix cache regular clean (#22540)
new 7e7e4da958 [exec](set_operation) Support one child node in set operation (#22463)
new 8754f629a6 [feature](executor) using max_instance_num to limit automatically instance (#22521)
new bd443bbac7 [bugfix](profile) not output some variables correctly (#22537)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/vec/exec/vset_operation_node.cpp | 16 +++++++++++-----
be/src/vec/exec/vset_operation_node.h | 2 +-
docs/en/docs/admin-manual/config/fe-config.md | 9 ---------
.../query-acceleration/pipeline-execution-engine.md | 1 +
docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 --------
.../query-acceleration/pipeline-execution-engine.md | 2 +-
.../src/main/java/org/apache/doris/common/Config.java | 7 -------
.../main/java/org/apache/doris/analysis/SetVar.java | 18 +-----------------
.../apache/doris/common/profile/SummaryProfile.java | 4 ++++
.../main/java/org/apache/doris/qe/SessionVariable.java | 7 ++++++-
10 files changed, 25 insertions(+), 49 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [bugfix](profile) not output some variables correctly (#22537)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bd443bbac757c6f16ccac695defbfa001d593fc2
Author: yiguolei <67...@qq.com>
AuthorDate: Thu Aug 3 13:17:02 2023 +0800
[bugfix](profile) not output some variables correctly (#22537)
---
.../src/main/java/org/apache/doris/common/profile/SummaryProfile.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index dec67d201f..ef656feaa8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -155,6 +155,10 @@ public class SummaryProfile {
for (String key : infos.keySet()) {
if (SUMMARY_KEYS.contains(key)) {
summaryProfile.addInfoString(key, infos.get(key));
+ } else if (EXECUTION_SUMMARY_KEYS.contains(key)) {
+ // Some static value is build in summary profile, should add
+ // them to execution summary profile during update.
+ executionSummaryProfile.addInfoString(key, infos.get(key));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [feature](executor) using max_instance_num to limit automatically instance (#22521)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8754f629a6c0dcd90634b8e46c881b602f5dd3bc
Author: Mryange <59...@users.noreply.github.com>
AuthorDate: Thu Aug 3 13:12:32 2023 +0800
[feature](executor) using max_instance_num to limit automatically instance (#22521)
---
docs/en/docs/admin-manual/config/fe-config.md | 9 ---------
.../query-acceleration/pipeline-execution-engine.md | 1 +
docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 --------
.../query-acceleration/pipeline-execution-engine.md | 2 +-
.../src/main/java/org/apache/doris/common/Config.java | 7 -------
.../main/java/org/apache/doris/analysis/SetVar.java | 18 +-----------------
.../main/java/org/apache/doris/qe/SessionVariable.java | 7 ++++++-
7 files changed, 9 insertions(+), 43 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index 3820d72fc4..224cb25a1f 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2723,15 +2723,6 @@ MasterOnly: false
Controls whether to enable query hit statistics. The default is false.
-#### `max_instance_num`
-
-<version since="dev"></version>
-
-Default: 128
-
-This is used to limit the setting of "parallel_fragment_exec_instance_num".
-"parallel_fragment_exec_instance_num" cannot be set higher than "max_instance_num".
-
#### `div_precision_increment`
<version since="dev"></version>
diff --git a/docs/en/docs/query-acceleration/pipeline-execution-engine.md b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
index d32ca42217..8d3a2697c1 100644
--- a/docs/en/docs/query-acceleration/pipeline-execution-engine.md
+++ b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
@@ -77,3 +77,4 @@ If the user upgrades from a lower version, the default value will be the paralle
```
set parallel_pipeline_task_num = 0;
```
+You can limit the automatically configured concurrency by setting "max_instance_num."(The default value is 64)
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index 9913de60d9..b93ca46774 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2714,14 +2714,6 @@ show data (其他用法:HELP SHOW DATA)
这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。
-#### `max_instance_num`
-
-<version since="dev"></version>
-
-默认值:128
-
-用于限制parallel_fragment_exec_instance_num的设置,set parallel_fragment_exec_instance_num不能超过max_instance_num
-
#### `enable_query_hit_stats`
<version since="dev"></version>
diff --git a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
index 0ce0f65448..3953d9c8d8 100644
--- a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
+++ b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
@@ -77,4 +77,4 @@ set enable_pipeline_engine = true;
```
set parallel_pipeline_task_num = 0;
```
-
+可以通过设置max_instance_num来限制自动设置的并发数(默认为64)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index c5d8765128..1a6456fefa 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1957,13 +1957,6 @@ public class Config extends ConfigBase {
"Now default set to true, not support create complex type(array/struct/map) nested complex type "
+ "when we create table, only support array type nested array"})
public static boolean disable_nested_complex_type = true;
- /*
- * "max_instance_num" is used to set the maximum concurrency. When the value set
- * by "parallel_fragment_exec_instance_num" is greater than "max_instance_num",
- * an error will be reported.
- */
- @ConfField(mutable = true)
- public static int max_instance_num = 128;
/*
* This variable indicates the number of digits by which to increase the scale
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
index a0d61248a5..6142feec89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
@@ -19,7 +19,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@@ -179,22 +178,7 @@ public class SetVar {
this.value = new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(getResult().getStringValue()));
this.result = (LiteralExpr) this.value;
}
- if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM)) {
- int instanceNum = Integer.parseInt(getResult().getStringValue());
- if (instanceNum > Config.max_instance_num) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR,
- SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM,
- instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")");
- }
- }
- if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_PIPELINE_TASK_NUM)) {
- int instanceNum = Integer.parseInt(getValue().getStringValue());
- if (instanceNum > Config.max_instance_num) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR,
- SessionVariable.PARALLEL_PIPELINE_TASK_NUM,
- instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")");
- }
- }
+
if (getVariable().equalsIgnoreCase(SessionVariable.EXEC_MEM_LIMIT)) {
this.value = new StringLiteral(Long.toString(ParseUtil.analyzeDataVolumn(getResult().getStringValue())));
this.result = (LiteralExpr) this.value;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 739f94cba7..4b4a4eba9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -106,6 +106,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num";
public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num";
+ public static final String MAX_INSTANCE_NUM = "max_instance_num";
public static final String ENABLE_INSERT_STRICT = "enable_insert_strict";
public static final String ENABLE_SPILLING = "enable_spilling";
public static final String ENABLE_EXCHANGE_NODE_PARALLEL_MERGE = "enable_exchange_node_parallel_merge";
@@ -563,6 +564,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true)
public int parallelPipelineTaskNum = 0;
+ @VariableMgr.VarAttr(name = MAX_INSTANCE_NUM)
+ public int maxInstanceNum = 64;
+
@VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true)
public boolean enableInsertStrict = true;
@@ -1428,7 +1432,8 @@ public class SessionVariable implements Serializable, Writable {
public int getParallelExecInstanceNum() {
if (enablePipelineEngine && parallelPipelineTaskNum == 0) {
int size = Env.getCurrentSystemInfo().getMinPipelineExecutorSize();
- return (size + 1) / 2;
+ int autoInstance = (size + 1) / 2;
+ return Math.min(autoInstance, maxInstanceNum);
} else if (enablePipelineEngine) {
return parallelPipelineTaskNum;
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [exec](set_operation) Support one child node in set operation (#22463)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7e7e4da958d3e522345c3d60c9cc3479d09cc3f5
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Thu Aug 3 10:35:59 2023 +0800
[exec](set_operation) Support one child node in set operation (#22463)
Support one child node in set operation
---
be/src/vec/exec/vset_operation_node.cpp | 16 +++++++++++-----
be/src/vec/exec/vset_operation_node.h | 2 +-
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp
index 17db664c8a..ac01cbd5da 100644
--- a/be/src/vec/exec/vset_operation_node.cpp
+++ b/be/src/vec/exec/vset_operation_node.cpp
@@ -218,7 +218,10 @@ Status VSetOperationNode<is_intersect>::alloc_resource(RuntimeState* state) {
for (const VExprContextSPtrs& exprs : _child_expr_lists) {
RETURN_IF_ERROR(VExpr::open(exprs, state));
}
- _probe_columns.resize(_child_expr_lists[1].size());
+ // Add the if check only for compatible with old optimiser
+ if (_child_expr_lists.size() > 1) {
+ _probe_columns.resize(_child_expr_lists[1].size());
+ }
return Status::OK();
}
@@ -326,8 +329,8 @@ void VSetOperationNode<is_intersect>::hash_table_init() {
bool has_null = false;
int key_byte_size = 0;
- _probe_key_sz.resize(_child_expr_lists[1].size());
_build_key_sz.resize(_child_expr_lists[0].size());
+ _probe_key_sz.resize(_child_expr_lists[0].size());
for (int i = 0; i < _child_expr_lists[0].size(); ++i) {
const auto vexpr = _child_expr_lists[0][i]->root();
const auto& data_type = vexpr->data_type();
@@ -407,6 +410,7 @@ Status VSetOperationNode<is_intersect>::sink(RuntimeState* state, Block* block,
*_hash_table_variants);
}
_build_finished = true;
+ _can_read = _children.size() == 1;
}
}
return Status::OK();
@@ -530,11 +534,14 @@ Status VSetOperationNode<is_intersect>::sink_probe(RuntimeState* state, int chil
*_hash_table_variants));
}
- return eos ? finalize_probe(state, child_id) : Status::OK();
+ if (eos) {
+ _finalize_probe(child_id);
+ }
+ return Status::OK();
}
template <bool is_intersect>
-Status VSetOperationNode<is_intersect>::finalize_probe(RuntimeState* /*state*/, int child_id) {
+void VSetOperationNode<is_intersect>::_finalize_probe(int child_id) {
if (child_id != (_children.size() - 1)) {
refresh_hash_table();
if constexpr (is_intersect) {
@@ -554,7 +561,6 @@ Status VSetOperationNode<is_intersect>::finalize_probe(RuntimeState* /*state*/,
_can_read = true;
}
_probe_finished_children_index[child_id] = true;
- return Status::OK();
}
template <bool is_intersect>
diff --git a/be/src/vec/exec/vset_operation_node.h b/be/src/vec/exec/vset_operation_node.h
index 39a74f899e..9b21fca26b 100644
--- a/be/src/vec/exec/vset_operation_node.h
+++ b/be/src/vec/exec/vset_operation_node.h
@@ -70,11 +70,11 @@ public:
Status pull(RuntimeState* state, Block* output_block, bool* eos) override;
Status sink_probe(RuntimeState* state, int child_id, Block* block, bool eos);
- Status finalize_probe(RuntimeState* state, int child_id);
bool is_child_finished(int child_id) const;
private:
+ void _finalize_probe(int child_id);
//Todo: In build process of hashtable, It's same as join node.
//It's time to abstract out the same methods and provide them directly to others;
void hash_table_init();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org