You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/24 03:34:59 UTC
(doris) branch master updated: [pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 96d29e14429 [pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328)
96d29e14429 is described below
commit 96d29e14429ff5415152333368141c538757a5ab
Author: Gabriel <ga...@gmail.com>
AuthorDate: Sat Feb 24 11:34:52 2024 +0800
[pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328)
---
.../java/org/apache/doris/datasource/FileQueryScanNode.java | 10 ++++++++++
.../org/apache/doris/datasource/tvf/source/TVFScanNode.java | 5 +++++
2 files changed, 15 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 6c60692ea21..7afb04831ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -503,6 +503,16 @@ public abstract class FileQueryScanNode extends FileScanNode {
return i;
}
+ @Override
+ public int getNumInstances() {
+ if (ConnectContext.get() != null
+ && ConnectContext.get().getSessionVariable().getEnablePipelineXEngine()
+ && ConnectContext.get().getSessionVariable().isIgnoreStorageDataDistribution()) {
+ return ConnectContext.get().getSessionVariable().getParallelExecInstanceNum();
+ }
+ return scanRangeLocations.size();
+ }
+
protected abstract TFileType getLocationType() throws UserException;
protected abstract TFileType getLocationType(String location) throws UserException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index 83aa49f5ab0..dcd248a9782 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -143,4 +143,9 @@ public class TVFScanNode extends FileQueryScanNode {
}
return splits;
}
+
+ @Override
+ public int getNumInstances() {
+ return scanRangeLocations.size();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org