You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/11 16:55:45 UTC
[doris] 28/47: [Fix](func numbers) Remove backend_nums argument of numbers function (#25200)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6f23a0f86bf47425a93ecdc33fa75cb9161819e3
Author: zhiqqqq <se...@163.com>
AuthorDate: Tue Oct 10 07:25:58 2023 -0500
[Fix](func numbers) Remove backend_nums argument of numbers function (#25200)
---
.../vec/exec/data_gen_functions/vnumbers_tvf.cpp | 3 ++
.../sql-functions/table-functions/numbers.md | 4 +--
.../sql-functions/table-functions/numbers.md | 4 +--
.../org/apache/doris/planner/DataGenScanNode.java | 10 +++++++
.../main/java/org/apache/doris/qe/Coordinator.java | 15 +++++++---
.../tablefunction/NumbersTableValuedFunction.java | 32 ++++++----------------
.../http_rest_api/post/test_query_stmt.groovy | 2 +-
.../nereids_syntax_p0/aggregate_strategies.groovy | 8 +++---
.../suites/nereids_syntax_p0/function.groovy | 4 +--
.../nereids_syntax_p2/aggregate_strategies.groovy | 8 +++---
10 files changed, 46 insertions(+), 44 deletions(-)
diff --git a/be/src/vec/exec/data_gen_functions/vnumbers_tvf.cpp b/be/src/vec/exec/data_gen_functions/vnumbers_tvf.cpp
index 278e9743320..2ac6c0fca42 100644
--- a/be/src/vec/exec/data_gen_functions/vnumbers_tvf.cpp
+++ b/be/src/vec/exec/data_gen_functions/vnumbers_tvf.cpp
@@ -80,6 +80,9 @@ Status VNumbersTVF::get_next(RuntimeState* state, vectorized::Block* block, bool
}
Status VNumbersTVF::set_scan_ranges(const std::vector<TScanRangeParams>& scan_range_params) {
+ // Currently we do not support multi-threads numbers function, so there is no need to
+ // use more than one scan_range_param.
+ DCHECK(scan_range_params.size() == 1);
_total_numbers =
scan_range_params[0].scan_range.data_gen_scan_range.numbers_params.totalNumbers;
return Status::OK();
diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/numbers.md b/docs/en/docs/sql-manual/sql-functions/table-functions/numbers.md
index c555e403acb..ae0f694e71e 100644
--- a/docs/en/docs/sql-manual/sql-functions/table-functions/numbers.md
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/numbers.md
@@ -36,14 +36,12 @@ This function is used in FROM clauses.
```sql
numbers(
- "number" = "n",
- "backend_num" = "m"
+ "number" = "n"
);
```
parameter：
- `number`: It means to generate rows [0, n).
-- `backend_num`: Optional parameters. It means this function is executed simultaneously on `m` be nodes (multiple BEs need to be deployed).
### example
```
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/numbers.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/numbers.md
index ce943795053..ed44c78eb49 100644
--- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/numbers.md
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/numbers.md
@@ -35,14 +35,12 @@ under the License.
#### syntax
```sql
numbers(
- "number" = "n",
- "backend_num" = "m"
+ "number" = "n"
);
```
参数：
- `number`: 代表生成[0,n)的行。
-- `backend_num`: 可选参数,代表`m`个be节点同时执行该函数（需要部署多个be）。
### example
```
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
index 46af5ec1ae2..d00641d135c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
@@ -107,4 +107,14 @@ public class DataGenScanNode extends ExternalScanNode {
public boolean needToCheckColumnPriv() {
return false;
}
+
+ // Currently DataGenScanNode is only used by DataGenTableValuedFunction, which is
+ // inherited by NumbersTableValuedFunction.
+ // NumbersTableValuedFunction is not a complete implementation for now, since its
+ // function signature do not support us to split total numbers, so it can not be executed
+ // by multi-processes or multi-threads. So we assign instance number to 1.
+ @Override
+ public int getNumInstances() {
+ return 1;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 2c52eecdd04..49afb6e6851 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -637,8 +637,9 @@ public class Coordinator implements CoordInterface {
// else use exec_plan_fragments directly.
// we choose #fragments >=2 because in some cases
// we need ensure that A fragment is already prepared to receive data before B fragment sends data.
- // For example: select * from numbers("10","w") will generate ExchangeNode and TableValuedFunctionScanNode,
- // we should ensure TableValuedFunctionScanNode does not send data until ExchangeNode is ready to receive.
+ // For example: select * from numbers("number"="10") will generate ExchangeNode and
+ // TableValuedFunctionScanNode, we should ensure TableValuedFunctionScanNode does
+ // not send data until ExchangeNode is ready to receive.
boolean twoPhaseExecution = fragments.size() >= 2;
for (PlanFragment fragment : fragments) {
FragmentExecParams params = fragmentExecParamsMap.get(fragment.getFragmentId());
@@ -759,8 +760,9 @@ public class Coordinator implements CoordInterface {
// else use exec_plan_fragments directly.
// we choose #fragments >=2 because in some cases
// we need ensure that A fragment is already prepared to receive data before B fragment sends data.
- // For example: select * from numbers("10","w") will generate ExchangeNode and TableValuedFunctionScanNode,
- // we should ensure TableValuedFunctionScanNode does not send data until ExchangeNode is ready to receive.
+ // For example: select * from numbers("number"="10") will generate ExchangeNode and
+ // TableValuedFunctionScanNode, we should ensure TableValuedFunctionScanNode does not
+ // send data until ExchangeNode is ready to receive.
boolean twoPhaseExecution = fragments.size() >= 2;
for (PlanFragment fragment : fragments) {
FragmentExecParams params = fragmentExecParamsMap.get(fragment.getFragmentId());
@@ -2116,6 +2118,11 @@ public class Coordinator implements CoordInterface {
FragmentScanRangeAssignment assignment,
Map<TNetworkAddress, Long> assignedBytesPerHost,
Map<TNetworkAddress, Long> replicaNumPerHost) throws Exception {
+ // Type of locations is List, it could have elements that have same "location"
+ // and we do have this situation for some scan node.
+ // The duplicate "location" will NOT be filtered by FragmentScanRangeAssignment,
+ // since FragmentScanRangeAssignment use List<TScanRangeParams> as its value type,
+ // duplicate "locations" will be converted to list.
for (TScanRangeLocations scanRangeLocations : locations) {
Reference<Long> backendIdRef = new Reference<Long>();
TScanRangeLocation minLocation = selectBackendsByRoundRobin(scanRangeLocations,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
index 639dfeef351..3c0b578b50a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/NumbersTableValuedFunction.java
@@ -41,20 +41,16 @@ import java.util.Map;
// have a single column number
/**
- * The Implement of table valued function——numbers("number" = "N", "backend_num" = "M").
+ * The Implement of table valued function——numbers("number" = "N").
*/
public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
public static final String NAME = "numbers";
public static final String NUMBER = "number";
- public static final String BACKEND_NUM = "backend_num";
private static final ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>()
.add(NUMBER)
- .add(BACKEND_NUM)
.build();
// The total numbers will be generated.
private long totalNumbers;
- // The total backends will server it.
- private int tabletsNum;
/**
* Constructor.
@@ -70,11 +66,6 @@ public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
validParams.put(key.toLowerCase(), params.get(key));
}
- try {
- tabletsNum = Integer.parseInt(validParams.getOrDefault(BACKEND_NUM, "1"));
- } catch (NumberFormatException e) {
- throw new AnalysisException("can not parse `backend_num` param to natural number");
- }
String numberStr = validParams.get(NUMBER);
if (!Strings.isNullOrEmpty(numberStr)) {
try {
@@ -92,10 +83,6 @@ public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
return totalNumbers;
}
- public int getTabletsNum() {
- return tabletsNum;
- }
-
@Override
public TDataGenFunctionName getDataGenFunctionName() {
return TDataGenFunctionName.NUMBERS;
@@ -124,17 +111,16 @@ public class NumbersTableValuedFunction extends DataGenTableValuedFunction {
if (backendList.isEmpty()) {
throw new AnalysisException("No Alive backends");
}
+
Collections.shuffle(backendList);
List<TableValuedFunctionTask> res = Lists.newArrayList();
- for (int i = 0; i < tabletsNum; ++i) {
- TScanRange scanRange = new TScanRange();
- TDataGenScanRange dataGenScanRange = new TDataGenScanRange();
- TTVFNumbersScanRange tvfNumbersScanRange = new TTVFNumbersScanRange();
- tvfNumbersScanRange.setTotalNumbers(totalNumbers);
- dataGenScanRange.setNumbersParams(tvfNumbersScanRange);
- scanRange.setDataGenScanRange(dataGenScanRange);
- res.add(new TableValuedFunctionTask(backendList.get(i % backendList.size()), scanRange));
- }
+ TScanRange scanRange = new TScanRange();
+ TDataGenScanRange dataGenScanRange = new TDataGenScanRange();
+ TTVFNumbersScanRange tvfNumbersScanRange = new TTVFNumbersScanRange();
+ tvfNumbersScanRange.setTotalNumbers(totalNumbers);
+ dataGenScanRange.setNumbersParams(tvfNumbersScanRange);
+ scanRange.setDataGenScanRange(dataGenScanRange);
+ res.add(new TableValuedFunctionTask(backendList.get(0), scanRange));
return res;
}
}
diff --git a/regression-test/suites/http_rest_api/post/test_query_stmt.groovy b/regression-test/suites/http_rest_api/post/test_query_stmt.groovy
index 7a03cdc48bd..bcf6e23984c 100644
--- a/regression-test/suites/http_rest_api/post/test_query_stmt.groovy
+++ b/regression-test/suites/http_rest_api/post/test_query_stmt.groovy
@@ -49,7 +49,7 @@ suite("test_query_stmt") {
def url= "/api/query/default_cluster/" + context.config.defaultDb
// test select
- def stmt1 = """ select * from numbers('number' = '10', 'backend_num' = '1') """
+ def stmt1 = """ select * from numbers('number' = '10') """
def stmt1_json = JsonOutput.toJson(new Stmt(stmt: stmt1));
def resJson = http_post(url, stmt1_json)
diff --git a/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy b/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy
index ea63b5b7898..e2f5d8e0cf0 100644
--- a/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy
+++ b/regression-test/suites/nereids_syntax_p0/aggregate_strategies.groovy
@@ -170,7 +170,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000', 'backend_num'='10')"""
+ from numbers('number' = '10000')"""
result([[10000L]])
}
@@ -178,7 +178,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000', 'backend_num'='10')"""
+ from numbers('number' = '10000')"""
result([[10000L]])
}
@@ -186,7 +186,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000', 'backend_num'='1')"""
+ from numbers('number' = '10000')"""
result([[10000L]])
}
@@ -194,7 +194,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000', 'backend_num'='1')"""
+ from numbers('number' = '10000')"""
result([[10000L]])
}
diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy b/regression-test/suites/nereids_syntax_p0/function.groovy
index 3f05deb09a2..498273bad91 100644
--- a/regression-test/suites/nereids_syntax_p0/function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/function.groovy
@@ -70,7 +70,7 @@ suite("nereids_function") {
// numbers: table valued function
test {
- sql "select `number` from numbers(number = 10, backend_num = 1)"
+ sql "select `number` from numbers(number = 10)"
result([[0L], [1L], [2L], [3L], [4L], [5L], [6L], [7L], [8L], [9L]])
}
@@ -91,7 +91,7 @@ suite("nereids_function") {
qt_subquery3 """ select a.number from numbers("number" = "10") a where number in (select number from numbers("number" = "10") b where a.number=b.number); """
test {
- sql """select `number` from numbers("number" = -1, 'backend_num' = `1`)"""
+ sql """select `number` from numbers("number" = "-1")"""
result([])
}
diff --git a/regression-test/suites/nereids_syntax_p2/aggregate_strategies.groovy b/regression-test/suites/nereids_syntax_p2/aggregate_strategies.groovy
index cebd90a955c..e2ad64f688e 100644
--- a/regression-test/suites/nereids_syntax_p2/aggregate_strategies.groovy
+++ b/regression-test/suites/nereids_syntax_p2/aggregate_strategies.groovy
@@ -20,7 +20,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000000', 'backend_num'='10')"""
+ from numbers('number' = '10000000')"""
result([[10000000L]])
}
@@ -28,7 +28,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000000', 'backend_num'='10')"""
+ from numbers('number' = '10000000')"""
result([[10000000L]])
}
@@ -36,7 +36,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000000', 'backend_num'='1')"""
+ from numbers('number' = '10000000')"""
result([[10000000L]])
}
@@ -44,7 +44,7 @@ suite("aggregate_strategies") {
sql """select
/*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT')*/
count(distinct number)
- from numbers('number' = '10000000', 'backend_num'='1')"""
+ from numbers('number' = '10000000')"""
result([[10000000L]])
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org