You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/10 07:26:25 UTC
(doris) branch 2.1-tmp updated (77ad3f6a193 -> b85bf3b6b06)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
from 77ad3f6a193 [feature](hive)Get updated information from coordinate and commit (#32441) (#33466)
new 0ab8b57db7c [enhance](mtmv)support create mtmv with other mtmv (#32984)
new 6798a24a27c [Enhencement](Nereids) reduce child output rows if agg child is literal (#32188)
new 8fd6d4c41b8 [Chore](build) add -Wconversion and remove some unused code (#33127)
new 2e40e395844 [chore](spill) add timers for performance tuning (#33185)
new 67bb5196137 [Fix](nereids) forward the user define variables to master (#33013)
new 5e59c09a602 [Fix](nereids) modify the binding aggregate function in order by (#32758)
new 9670422d61e [fix](inverted index) fix the incorrect result issue of COUNT_ON_INDEX for key columns (#33164)
new edd17019633 [fix](Nereids) convert agg state type failed in some cases (#33208)
new b696909775e [fix](plsql) Fix plsql variable initialization (#33186)
new 2b1ab89b5ba [fix](memory) Fix memory log compile by ASAN (#33162)
new 96867ff3fd7 [fix](Nereids) support update without filter (#33214)
new 7fae123b01f [FIX](inverted_index) fix inverted index write array with _doc is empty (#33170)
new b85bf3b6b06 [test](cast) add test for stream load cast (#33189)
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/CMakeLists.txt | 11 +-
be/src/clucene | 2 +-
be/src/exprs/bloom_filter_func.h | 4 +-
be/src/glibc-compatibility/CMakeLists.txt | 1 +
be/src/http/http_client.h | 2 +-
be/src/olap/cumulative_compaction_policy.cpp | 2 +-
be/src/olap/memtable.cpp | 4 +-
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/rowset/segment_v2/bloom_filter.cpp | 2 +-
.../rowset/segment_v2/inverted_index_cache.cpp | 4 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 9 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 20 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 5 +-
be/src/olap/wal/wal_table.cpp | 15 +-
.../exec/partitioned_aggregation_sink_operator.cpp | 5 +-
.../partitioned_aggregation_source_operator.cpp | 7 +-
.../exec/partitioned_hash_join_probe_operator.cpp | 100 ++-
.../exec/partitioned_hash_join_probe_operator.h | 24 +-
.../exec/partitioned_hash_join_sink_operator.cpp | 53 +-
.../exec/partitioned_hash_join_sink_operator.h | 12 +-
be/src/pipeline/exec/set_probe_sink_operator.cpp | 4 +-
be/src/pipeline/exec/spill_sort_sink_operator.cpp | 10 +-
.../pipeline/exec/spill_sort_source_operator.cpp | 9 +-
be/src/pipeline/pipeline_x/operator.h | 6 +
be/src/pipeline/pipeline_x/pipeline_x_task.cpp | 2 +-
be/src/pipeline/task_queue.cpp | 2 +-
be/src/pipeline/task_queue.h | 2 +-
be/src/runtime/decimalv2_value.cpp | 2 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 8 +-
be/src/runtime/workload_group/workload_group.cpp | 2 +-
.../workload_group/workload_group_manager.cpp | 4 +-
be/src/util/counts.h | 4 +-
be/src/util/cpu_info.cpp | 2 +-
be/src/util/date_func.cpp | 4 +-
be/src/util/hash_util.hpp | 2 +-
be/src/util/mem_info.cpp | 21 +-
be/src/util/os_util.cpp | 6 +-
be/src/util/parse_util.cpp | 6 +-
be/src/util/perf_counters.cpp | 4 +
be/src/util/runtime_profile.cpp | 2 +-
be/src/util/url_coding.cpp | 2 +-
.../aggregate_function_avg_weighted.h | 2 +-
.../aggregate_function_stddev.h | 2 +-
be/src/vec/columns/column.h | 7 -
be/src/vec/columns/column_array.h | 4 -
be/src/vec/columns/column_complex.h | 6 -
be/src/vec/columns/column_const.cpp | 16 -
be/src/vec/columns/column_const.h | 2 -
be/src/vec/columns/column_decimal.h | 5 -
be/src/vec/columns/column_dictionary.h | 6 -
be/src/vec/columns/column_dummy.h | 14 -
be/src/vec/columns/column_fixed_length_object.h | 6 -
be/src/vec/columns/column_impl.h | 27 -
be/src/vec/columns/column_map.h | 3 -
be/src/vec/columns/column_nullable.h | 4 -
be/src/vec/columns/column_object.h | 5 -
be/src/vec/columns/column_string.h | 4 -
be/src/vec/columns/column_struct.cpp | 23 -
be/src/vec/columns/column_struct.h | 1 -
be/src/vec/columns/column_vector.h | 17 +-
be/src/vec/columns/predicate_column.h | 6 -
be/src/vec/common/uint128.h | 7 +
be/src/vec/core/types.h | 6 +-
be/src/vec/data_types/data_type.h | 2 +-
be/src/vec/data_types/data_type_decimal.h | 3 +-
.../vec/data_types/serde/data_type_number_serde.h | 2 +-
be/src/vec/exec/scan/scanner_context.cpp | 4 +-
be/src/vec/exec/vset_operation_node.cpp | 4 +-
.../exprs/table_function/vexplode_json_array.cpp | 2 +-
be/src/vec/functions/array/function_array_join.h | 2 +-
be/src/vec/functions/function_cast.h | 12 +-
be/src/vec/functions/function_quantile_state.cpp | 2 +-
be/src/vec/functions/int_div.h | 6 +-
be/src/vec/sink/writer/vtablet_writer.cpp | 8 +-
be/test/olap/wal/wal_manager_test.cpp | 4 +-
be/test/vec/core/block_spill_test.cpp | 8 +-
be/test/vec/core/block_test.cpp | 4 +-
.../serde/data_type_serde_arrow_test.cpp | 2 +-
.../serde/data_type_serde_mysql_test.cpp | 2 +-
.../data_types/serde/data_type_serde_pb_test.cpp | 2 +-
.../vec/data_types/serde/data_type_serde_test.cpp | 2 +-
be/test/vec/jsonb/serialize_test.cpp | 2 +-
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../antlr4/org/apache/doris/nereids/PLLexer.g4 | 4 -
.../antlr4/org/apache/doris/nereids/PLParser.g4 | 2 +-
.../org/apache/doris/analysis/LiteralExpr.java | 20 +
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/BindExpression.java | 57 +-
.../nereids/rules/analysis/FillUpMissingSlots.java | 2 +-
.../expression/rules/ConvertAggStateCast.java | 38 +-
.../rewrite/ReduceAggregateChildOutputRows.java | 89 ++
.../trees/plans/commands/info/CreateMTMVInfo.java | 11 -
.../src/main/java/org/apache/doris/plsql/Exec.java | 6 -
.../doris/plsql/executor/DorisRowResult.java | 2 +-
.../java/org/apache/doris/qe/ConnectContext.java | 8 +
.../java/org/apache/doris/qe/ConnectProcessor.java | 25 +-
.../java/org/apache/doris/qe/MasterOpExecutor.java | 17 +-
.../rules/analysis/FillUpMissingSlotsTest.java | 2 +-
gensrc/thrift/FrontendService.thrift | 3 +-
.../data/inverted_index_p0/test_index_delete.out | 16 +
.../data/load_p0/stream_load/test_cast1.csv | 1 +
.../data/load_p0/stream_load/test_cast1.json | 1 +
.../data/load_p0/stream_load/test_cast2.csv | 1 +
.../data/load_p0/stream_load/test_cast2.json | 1 +
.../data/load_p0/stream_load/test_cast3.csv | 1 +
.../data/load_p0/stream_load/test_cast3.json | 1 +
.../load_p0/stream_load/test_stream_load_cast.out | 19 +
.../data/mtmv_p0/test_multi_level_mtmv.out | 13 +
.../eliminate_aggregate_constant.out | 993 +++++++++++++++++++++
.../nereids_syntax_p0/order_by_bind_priority.out | 25 +
.../data/plsql_p0/test_plsql_variable.out | 13 +
.../inverted_index_p0/test_index_delete.groovy | 65 ++
.../stream_load/test_stream_load_cast.groovy | 336 +++++++
.../suites/mtmv_p0/test_build_mtmv.groovy | 15 -
.../suites/mtmv_p0/test_multi_level_mtmv.groovy | 81 ++
.../eliminate_aggregate_constant.groovy | 285 ++++++
.../order_by_bind_priority.groovy | 38 +
.../suites/plsql_p0/test_plsql_loop_cursor.groovy | 7 +
.../suites/plsql_p0/test_plsql_variable.groovy | 110 +++
121 files changed, 2546 insertions(+), 403 deletions(-)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReduceAggregateChildOutputRows.java
create mode 100644 regression-test/data/inverted_index_p0/test_index_delete.out
create mode 100644 regression-test/data/load_p0/stream_load/test_cast1.csv
create mode 100644 regression-test/data/load_p0/stream_load/test_cast1.json
create mode 100644 regression-test/data/load_p0/stream_load/test_cast2.csv
create mode 100644 regression-test/data/load_p0/stream_load/test_cast2.json
create mode 100644 regression-test/data/load_p0/stream_load/test_cast3.csv
create mode 100644 regression-test/data/load_p0/stream_load/test_cast3.json
create mode 100644 regression-test/data/load_p0/stream_load/test_stream_load_cast.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_level_mtmv.out
create mode 100644 regression-test/data/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.out
create mode 100644 regression-test/data/nereids_syntax_p0/order_by_bind_priority.out
create mode 100644 regression-test/data/plsql_p0/test_plsql_variable.out
create mode 100644 regression-test/suites/inverted_index_p0/test_index_delete.groovy
create mode 100644 regression-test/suites/load_p0/stream_load/test_stream_load_cast.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
create mode 100644 regression-test/suites/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.groovy
create mode 100644 regression-test/suites/nereids_syntax_p0/order_by_bind_priority.groovy
create mode 100644 regression-test/suites/plsql_p0/test_plsql_variable.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/13: [fix](memory) Fix memory log compile by ASAN (#33162)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2b1ab89b5ba8b9a2004f764b1258bd33ec6cb871
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Wed Apr 3 19:00:15 2024 +0800
[fix](memory) Fix memory log compile by ASAN (#33162)
ASAN compiles BE, add markers in memory logs
---
be/src/runtime/memory/mem_tracker_limiter.cpp | 4 ++++
be/src/util/mem_info.cpp | 9 +++++++++
be/src/util/perf_counters.cpp | 4 ++++
3 files changed, 17 insertions(+)
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 84b80699431..08d8ea4d5e2 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -202,7 +202,11 @@ void MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
snapshot.peak_consumption = -1;
(*snapshots).emplace_back(snapshot);
+#ifdef ADDRESS_SANITIZER
+ snapshot.type = "[ASAN]process resident memory"; // from /proc VmRSS VmHWM
+#else
snapshot.type = "process resident memory"; // from /proc VmRSS VmHWM
+#endif
snapshot.label = "";
snapshot.limit = -1;
snapshot.cur_consumption = PerfCounters::get_vm_rss();
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 823490942af..a36acddc139 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -393,6 +393,15 @@ void MemInfo::refresh_proc_meminfo() {
_s_sys_mem_available.store(_mem_info_bytes["MemAvailable"], std::memory_order_relaxed);
_s_sys_mem_available_str = PrettyPrinter::print(
_s_sys_mem_available.load(std::memory_order_relaxed), TUnit::BYTES);
+#ifdef ADDRESS_SANITIZER
+ _s_sys_mem_available_str =
+ "[ASAN]" +
+ PrettyPrinter::print(_s_sys_mem_available.load(std::memory_order_relaxed),
+ TUnit::BYTES);
+#else
+ _s_sys_mem_available_str = PrettyPrinter::print(
+ _s_sys_mem_available.load(std::memory_order_relaxed), TUnit::BYTES);
+#endif
}
}
diff --git a/be/src/util/perf_counters.cpp b/be/src/util/perf_counters.cpp
index 67a9c013e1d..18fb9766ada 100644
--- a/be/src/util/perf_counters.cpp
+++ b/be/src/util/perf_counters.cpp
@@ -585,7 +585,11 @@ void PerfCounters::refresh_proc_status() {
_vm_size = parse_bytes("status/VmSize");
_vm_peak = parse_bytes("status/VmPeak");
_vm_rss = parse_bytes("status/VmRSS");
+#ifdef ADDRESS_SANITIZER
+ _vm_rss_str = "[ASAN]" + PrettyPrinter::print(_vm_rss, TUnit::BYTES);
+#else
_vm_rss_str = PrettyPrinter::print(_vm_rss, TUnit::BYTES);
+#endif
_vm_hwm = parse_bytes("status/VmHWM");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/13: [test](cast) add test for stream load cast (#33189)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b85bf3b6b06d0c671c8f8c8b58e6e1b656a9393d
Author: HHoflittlefish777 <77...@users.noreply.github.com>
AuthorDate: Wed Apr 3 21:04:44 2024 +0800
[test](cast) add test for stream load cast (#33189)
---
.../data/load_p0/stream_load/test_cast1.csv | 1 +
.../data/load_p0/stream_load/test_cast1.json | 1 +
.../data/load_p0/stream_load/test_cast2.csv | 1 +
.../data/load_p0/stream_load/test_cast2.json | 1 +
.../data/load_p0/stream_load/test_cast3.csv | 1 +
.../data/load_p0/stream_load/test_cast3.json | 1 +
.../load_p0/stream_load/test_stream_load_cast.out | 19 ++
.../stream_load/test_stream_load_cast.groovy | 336 +++++++++++++++++++++
8 files changed, 361 insertions(+)
diff --git a/regression-test/data/load_p0/stream_load/test_cast1.csv b/regression-test/data/load_p0/stream_load/test_cast1.csv
new file mode 100644
index 00000000000..e7fe19bce20
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast1.csv
@@ -0,0 +1 @@
+3.12,3.12,3.12,3.12,3.12
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_cast1.json b/regression-test/data/load_p0/stream_load/test_cast1.json
new file mode 100644
index 00000000000..3c88572d34e
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast1.json
@@ -0,0 +1 @@
+{"k0": 3.12, "k1": 3.12, "k2": 3.12, "k3": 3.12, "k4": 3.12}
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_cast2.csv b/regression-test/data/load_p0/stream_load/test_cast2.csv
new file mode 100644
index 00000000000..dab93718a8a
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast2.csv
@@ -0,0 +1 @@
+"3.12","3.12","3.12","3.12","3.12","3.12","3.12","3.12","2024-4-2","2024-4-2","2024-4-2 17:00:00","2024-4-2 17:00:00"
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_cast2.json b/regression-test/data/load_p0/stream_load/test_cast2.json
new file mode 100644
index 00000000000..6ba01a409e6
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast2.json
@@ -0,0 +1 @@
+{"k0": "3.12", "k1": "3.12", "k2": "3.12", "k3": "3.12", "k4": "3.12","k5": "3.12","k6": "3.12","k7": "3.12","k8": "2024-4-2","k9": "2024-4-2","k10": "2024-4-2 17:00:00","k11": "2024-4-2 17:00:00"}
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_cast3.csv b/regression-test/data/load_p0/stream_load/test_cast3.csv
new file mode 100644
index 00000000000..f4e1ab1e6cb
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast3.csv
@@ -0,0 +1 @@
+2147483648,129,32768,9223372036854775808,2e+128,3.40282e+39,1.79769e+309,100000000.11111111111,2024-14-2,2024-4-222,2024-14-2 17:00:00,2024-4-2 27:00:00
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_cast3.json b/regression-test/data/load_p0/stream_load/test_cast3.json
new file mode 100644
index 00000000000..6f41ac81e3b
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_cast3.json
@@ -0,0 +1 @@
+{"k0": 2147483648, "k1": 128, "k2": 32768, "k3": 9223372036854775808, "k4": 2e+128,"k5": 3.40282e+39,"k6": 1.79769e+309,"k7": 100000000.11111111111,"k8": "2024-14-2","k9": "2024-4-222","k10": "2024-14-2 17:00:00","k11": "2024-4-2 27:00:00"}
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_stream_load_cast.out b/regression-test/data/load_p0/stream_load/test_stream_load_cast.out
new file mode 100644
index 00000000000..23256d7aeb0
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_stream_load_cast.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+\N \N \N \N \N \N \N \N \N \N \N \N
+
+-- !sql2 --
+\N \N \N \N \N \N \N \N \N \N \N \N
+
+-- !sql3 --
+\N \N \N \N \N \N \N \N \N \N \N \N
+
+-- !sql4 --
+\N \N \N \N \N 3.12 3.12 3.1 2024-04-02 2024-04-02 2024-04-02T17:00 2024-04-02T17:00
+
+-- !sql5 --
+\N \N \N \N \N \N \N 99999999.9 \N \N \N \N
+
+-- !sql6 --
+\N \N \N \N \N \N \N 99999999.9 \N \N \N \N
+
diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_cast.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_cast.groovy
new file mode 100644
index 00000000000..1f3c8e24f78
--- /dev/null
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load_cast.groovy
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_stream_load_cast", "p0") {
+ def tableName = "test_stream_load_cast"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ k0 INT NULL,
+ k1 TINYINT NULL,
+ k2 SMALLINT NULL,
+ k3 BIGINT NULL,
+ k4 LARGEINT NULL,
+ k5 FLOAT NULL,
+ k6 DOUBLE NULL,
+ k7 DECIMAL(9,1) NULL,
+ k8 DATE NULL,
+ k9 DATEV2 NULL,
+ k10 DATETIME NULL,
+ k11 DATETIMEV2 NULL,
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 32
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ // test decimal cast to integer
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k0,k1,k2,k3,k4'
+ set 'strict_mode', 'false'
+
+ file 'test_cast1.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql1 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k0,k1,k2,k3,k4'
+ set 'strict_mode', 'true'
+
+ file 'test_cast1.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k0,k1,k2,k3,k4'
+ set 'strict_mode', 'false'
+ set 'format', 'json'
+
+ file 'test_cast1.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql2 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'k0,k1,k2,k3,k4'
+ set 'strict_mode', 'true'
+ set 'format', 'json'
+
+ file 'test_cast1.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+
+ // test invaild
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'false'
+
+ file 'test_cast2.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql3 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'true'
+
+ file 'test_cast2.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'false'
+ set 'format', 'json'
+
+ file 'test_cast2.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql4 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'true'
+ set 'format', 'json'
+
+ file 'test_cast2.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+
+ // test over limit
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'false'
+
+ file 'test_cast3.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql5 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'true'
+
+ file 'test_cast3.csv'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'false'
+ set 'format', 'json'
+
+ file 'test_cast3.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ sql "sync"
+ qt_sql6 "select * from ${tableName}"
+ sql "sync"
+ sql "truncate table ${tableName}"
+ sql "sync"
+
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'strict_mode', 'true'
+ set 'format', 'json'
+
+ file 'test_cast3.json'
+ time 10000 // limit inflight 10s
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(1, json.NumberTotalRows)
+ assertEquals(1, json.NumberFilteredRows)
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/13: [Chore](build) add -Wconversion and remove some unused code (#33127)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8fd6d4c41b8bb1ea7b9b4acb7aa52eb986ddc925
Author: Pxl <px...@qq.com>
AuthorDate: Wed Apr 3 11:57:41 2024 +0800
[Chore](build) add -Wconversion and remove some unused code (#33127)
add -Wconversion and remove some unused code
---
be/CMakeLists.txt | 11 ++++++---
be/src/exprs/bloom_filter_func.h | 4 ++--
be/src/glibc-compatibility/CMakeLists.txt | 1 +
be/src/http/http_client.h | 2 +-
be/src/olap/cumulative_compaction_policy.cpp | 2 +-
be/src/olap/memtable.cpp | 4 ++--
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/rowset/segment_v2/bloom_filter.cpp | 2 +-
.../rowset/segment_v2/inverted_index_cache.cpp | 4 ++--
be/src/olap/wal/wal_table.cpp | 15 ++++++------
be/src/pipeline/exec/set_probe_sink_operator.cpp | 4 ++--
be/src/pipeline/pipeline_x/pipeline_x_task.cpp | 2 +-
be/src/pipeline/task_queue.cpp | 2 +-
be/src/pipeline/task_queue.h | 2 +-
be/src/runtime/decimalv2_value.cpp | 2 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 4 ++--
be/src/runtime/workload_group/workload_group.cpp | 2 +-
.../workload_group/workload_group_manager.cpp | 4 ++--
be/src/util/counts.h | 4 ++--
be/src/util/cpu_info.cpp | 2 +-
be/src/util/date_func.cpp | 4 ++--
be/src/util/hash_util.hpp | 2 +-
be/src/util/mem_info.cpp | 12 +++++-----
be/src/util/os_util.cpp | 6 ++---
be/src/util/parse_util.cpp | 6 ++---
be/src/util/runtime_profile.cpp | 2 +-
be/src/util/url_coding.cpp | 2 +-
.../aggregate_function_avg_weighted.h | 2 +-
.../aggregate_function_stddev.h | 2 +-
be/src/vec/columns/column.h | 7 ------
be/src/vec/columns/column_array.h | 4 ----
be/src/vec/columns/column_complex.h | 6 -----
be/src/vec/columns/column_const.cpp | 16 -------------
be/src/vec/columns/column_const.h | 2 --
be/src/vec/columns/column_decimal.h | 5 ----
be/src/vec/columns/column_dictionary.h | 6 -----
be/src/vec/columns/column_dummy.h | 14 -----------
be/src/vec/columns/column_fixed_length_object.h | 6 -----
be/src/vec/columns/column_impl.h | 27 ----------------------
be/src/vec/columns/column_map.h | 3 ---
be/src/vec/columns/column_nullable.h | 4 ----
be/src/vec/columns/column_object.h | 5 ----
be/src/vec/columns/column_string.h | 4 ----
be/src/vec/columns/column_struct.cpp | 23 ------------------
be/src/vec/columns/column_struct.h | 1 -
be/src/vec/columns/column_vector.h | 17 +++++++-------
be/src/vec/columns/predicate_column.h | 6 -----
be/src/vec/common/uint128.h | 7 ++++++
be/src/vec/core/types.h | 6 ++---
be/src/vec/data_types/data_type.h | 2 +-
be/src/vec/data_types/data_type_decimal.h | 3 ++-
.../vec/data_types/serde/data_type_number_serde.h | 2 +-
be/src/vec/exec/scan/scanner_context.cpp | 4 ++--
be/src/vec/exec/vset_operation_node.cpp | 4 ++--
.../exprs/table_function/vexplode_json_array.cpp | 2 +-
be/src/vec/functions/array/function_array_join.h | 2 +-
be/src/vec/functions/function_cast.h | 12 +++++-----
be/src/vec/functions/function_quantile_state.cpp | 2 +-
be/src/vec/functions/int_div.h | 6 ++---
be/src/vec/sink/writer/vtablet_writer.cpp | 8 +++----
be/test/olap/wal/wal_manager_test.cpp | 4 ++--
be/test/vec/core/block_spill_test.cpp | 8 +++----
be/test/vec/core/block_test.cpp | 4 ++--
.../serde/data_type_serde_arrow_test.cpp | 2 +-
.../serde/data_type_serde_mysql_test.cpp | 2 +-
.../data_types/serde/data_type_serde_pb_test.cpp | 2 +-
.../vec/data_types/serde/data_type_serde_test.cpp | 2 +-
be/test/vec/jsonb/serialize_test.cpp | 2 +-
68 files changed, 115 insertions(+), 240 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 9b72bc61777..f5d99c69962 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -279,9 +279,14 @@ if (COMPILER_CLANG)
-Wunused-volatile-lvalue
-Wunused-template
-Wunused-member-function
- -Wunused-macros)
- add_compile_options(-Wno-vla-extension)
- add_compile_options(-Wno-gnu-statement-expression)
+ -Wunused-macros
+ -Wconversion)
+ add_compile_options(-Wno-vla-extension
+ -Wno-gnu-statement-expression
+ -Wno-implicit-float-conversion
+ -Wno-implicit-int-conversion
+ -Wno-sign-conversion
+ -Wno-shorten-64-to-32)
if (USE_LIBCPP)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
if (NOT OS_MACOSX)
diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index 052893b0c0f..1473d4a4288 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -42,12 +42,12 @@ public:
Status merge(BloomFilterAdaptor* other) { return _bloom_filter->merge(*other->_bloom_filter); }
Status init(int len) {
- int log_space = log2(len);
+ int log_space = (int)log2(len);
return _bloom_filter->init(log_space, /*hash_seed*/ 0);
}
Status init(butil::IOBufAsZeroCopyInputStream* data, const size_t data_size) {
- int log_space = log2(data_size);
+ int log_space = (int)log2(data_size);
return _bloom_filter->init_from_directory(log_space, data, data_size, false, 0);
}
diff --git a/be/src/glibc-compatibility/CMakeLists.txt b/be/src/glibc-compatibility/CMakeLists.txt
index a29e712f4a0..c605e83951f 100644
--- a/be/src/glibc-compatibility/CMakeLists.txt
+++ b/be/src/glibc-compatibility/CMakeLists.txt
@@ -70,6 +70,7 @@ if (GLIBC_COMPATIBILITY)
-Wno-unused-command-line-argument
-Wno-unused-but-set-variable
-Wno-unused-macros
+ -Wno-conversion
)
elseif (COMPILER_GCC)
target_compile_options(
diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h
index c80d0c6f56a..d80f484ce80 100644
--- a/be/src/http/http_client.h
+++ b/be/src/http/http_client.h
@@ -109,7 +109,7 @@ public:
"actrual is : {}",
cl));
}
- *length = cl;
+ *length = (uint64_t)cl;
return Status::OK();
}
return Status::InternalError("failed to get content length. err code: {}", code);
diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index f8821748e6b..0d35ae7ca4f 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -121,7 +121,7 @@ void SizeBasedCumulativeCompactionPolicy::_calc_promotion_size(Tablet* tablet,
RowsetMetaSharedPtr base_rowset_meta,
int64_t* promotion_size) {
int64_t base_size = base_rowset_meta->total_disk_size();
- *promotion_size = base_size * _promotion_ratio;
+ *promotion_size = int64_t(base_size * _promotion_ratio);
// promotion_size is between _promotion_size and _promotion_min_size
if (*promotion_size >= _promotion_size) {
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 7cddd2abe01..87ae20237e5 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -210,8 +210,8 @@ void MemTable::insert(const vectorized::Block* input_block, const std::vector<ui
}
auto block_size1 = _input_mutable_block.allocated_bytes();
g_memtable_input_block_allocated_size << block_size1 - block_size0;
- size_t input_size = target_block.bytes() * num_rows / target_block.rows() *
- config::memtable_insert_memory_ratio;
+ auto input_size = size_t(target_block.bytes() * num_rows / target_block.rows() *
+ config::memtable_insert_memory_ratio);
_mem_usage += input_size;
_insert_mem_tracker->consume(input_size);
for (int i = 0; i < num_rows; i++) {
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index d3150c17e14..a2b5bf56de9 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -331,7 +331,7 @@ void StorageEngine::_garbage_sweeper_thread_callback() {
// when usage = 0.88, ratio is approximately 0.0057.
double ratio = (1.1 * (pi / 2 - std::atan(usage * 100 / 5 - 14)) - 0.28) / pi;
ratio = ratio > 0 ? ratio : 0;
- uint32_t curr_interval = max_interval * ratio;
+ auto curr_interval = uint32_t(max_interval * ratio);
curr_interval = std::max(curr_interval, min_interval);
curr_interval = std::min(curr_interval, max_interval);
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.cpp b/be/src/olap/rowset/segment_v2/bloom_filter.cpp
index 12a7f2395e7..19e2b93d29c 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter.cpp
@@ -53,7 +53,7 @@ uint32_t BloomFilter::used_bits(uint64_t value) {
uint32_t BloomFilter::optimal_bit_num(uint64_t n, double fpp) {
// ref parquet bloom_filter branch(BlockSplitBloomFilter.java)
- uint32_t num_bits = -8 * (double)n / log(1 - pow(fpp, 1.0 / 8));
+ auto num_bits = uint32_t(-8 * (double)n / log(1 - pow(fpp, 1.0 / 8)));
uint32_t max_bits = MAXIMUM_BYTES << 3;
if (num_bits > max_bits) {
num_bits = max_bits;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
index adbce0b4064..f6a0951b44c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
@@ -50,9 +50,9 @@ InvertedIndexSearcherCache::InvertedIndexSearcherCache(size_t capacity, uint32_t
static constexpr size_t fd_bound = 100000;
size_t search_limit_percent = config::inverted_index_fd_number_limit_percent;
if (fd_number <= fd_bound) {
- search_limit_percent *= 0.25; // default 10%
+ search_limit_percent = size_t(search_limit_percent * 0.25); // default 10%
} else if (fd_number > fd_bound && fd_number < fd_bound * 5) {
- search_limit_percent *= 0.5; // default 20%
+ search_limit_percent = size_t(search_limit_percent * 0.5); // default 20%
}
uint64_t open_searcher_limit = fd_number * search_limit_percent / 100;
diff --git a/be/src/olap/wal/wal_table.cpp b/be/src/olap/wal/wal_table.cpp
index 641ef8c6647..ec0c412379a 100644
--- a/be/src/olap/wal/wal_table.cpp
+++ b/be/src/olap/wal/wal_table.cpp
@@ -145,15 +145,16 @@ bool WalTable::_need_replay(std::shared_ptr<WalInfo> wal_info) {
return true;
}
#ifndef BE_TEST
- auto replay_interval = 0;
+ int64_t replay_interval = 0;
if (wal_info->get_retry_num() >= config::group_commit_replay_wal_retry_num) {
- replay_interval = pow(2, config::group_commit_replay_wal_retry_num) *
- config::group_commit_replay_wal_retry_interval_seconds * 1000 +
- (wal_info->get_retry_num() - config::group_commit_replay_wal_retry_num) *
- config::group_commit_replay_wal_retry_interval_max_seconds * 1000;
+ replay_interval =
+ int64_t(pow(2, config::group_commit_replay_wal_retry_num) *
+ config::group_commit_replay_wal_retry_interval_seconds * 1000 +
+ (wal_info->get_retry_num() - config::group_commit_replay_wal_retry_num) *
+ config::group_commit_replay_wal_retry_interval_max_seconds * 1000);
} else {
- replay_interval = pow(2, wal_info->get_retry_num()) *
- config::group_commit_replay_wal_retry_interval_seconds * 1000;
+ replay_interval = int64_t(pow(2, wal_info->get_retry_num()) *
+ config::group_commit_replay_wal_retry_interval_seconds * 1000);
}
return UnixMillis() - wal_info->get_start_time_ms() >= replay_interval;
#else
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index 2d7bcd42c2b..76248f2c75c 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -238,8 +238,8 @@ void SetProbeSinkOperatorX<is_intersect>::_refresh_hash_table(
bool is_need_shrink =
arg.hash_table->should_be_shrink(valid_element_in_hash_tbl);
if (is_intersect || is_need_shrink) {
- tmp_hash_table->init_buf_size(
- valid_element_in_hash_tbl / arg.hash_table->get_factor() + 1);
+ tmp_hash_table->init_buf_size(size_t(
+ valid_element_in_hash_tbl / arg.hash_table->get_factor() + 1));
}
arg.init_iterator();
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
index 4b85df05484..4bca88ec931 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
@@ -327,7 +327,7 @@ bool PipelineXTask::should_revoke_memory(RuntimeState* state, int64_t revocable_
DCHECK(big_memory_operator_num >= 0);
int64_t mem_limit_of_op;
if (0 == big_memory_operator_num) {
- mem_limit_of_op = (double)query_weighted_limit * 0.8;
+ mem_limit_of_op = int64_t(query_weighted_limit * 0.8);
} else {
mem_limit_of_op = query_weighted_limit / big_memory_operator_num;
}
diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp
index a79d865821d..617cd7a78d1 100644
--- a/be/src/pipeline/task_queue.cpp
+++ b/be/src/pipeline/task_queue.cpp
@@ -71,7 +71,7 @@ PipelineTask* PriorityTaskQueue::_try_take_unprotected(bool is_steal) {
}
}
DCHECK(level != -1);
- _queue_level_min_vruntime = min_vruntime;
+ _queue_level_min_vruntime = uint64_t(min_vruntime);
auto task = _sub_queues[level].try_take(is_steal);
if (task) {
diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h
index a6799fdb0c6..02994511019 100644
--- a/be/src/pipeline/task_queue.h
+++ b/be/src/pipeline/task_queue.h
@@ -76,7 +76,7 @@ public:
void inc_runtime(uint64_t delta_time) { _runtime += delta_time; }
- void adjust_runtime(uint64_t vruntime) { this->_runtime = vruntime * _level_factor; }
+ void adjust_runtime(uint64_t vruntime) { this->_runtime = uint64_t(vruntime * _level_factor); }
bool empty() { return _queue.empty(); }
diff --git a/be/src/runtime/decimalv2_value.cpp b/be/src/runtime/decimalv2_value.cpp
index 6155372ae6d..f3466ca47b1 100644
--- a/be/src/runtime/decimalv2_value.cpp
+++ b/be/src/runtime/decimalv2_value.cpp
@@ -328,7 +328,7 @@ static double sqrt_fractional(int128_t sqrt_int, int128_t remainder) {
const int128_t DecimalV2Value::SQRT_MOLECULAR_MAGNIFICATION = get_scale_base(PRECISION / 2);
const int128_t DecimalV2Value::SQRT_DENOMINATOR =
- std::sqrt(ONE_BILLION) * get_scale_base(PRECISION / 2 - SCALE);
+ int128_t(std::sqrt(ONE_BILLION) * get_scale_base(PRECISION / 2 - SCALE));
DecimalV2Value DecimalV2Value::sqrt(const DecimalV2Value& v) {
int128_t x = v.value();
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 3edbd21c745..84b80699431 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -606,9 +606,9 @@ int64_t MemTrackerLimiter::free_top_overcommit_query(
tracker->consumption()));
continue;
}
- int64_t overcommit_ratio =
+ auto overcommit_ratio = int64_t(
(static_cast<double>(tracker->consumption()) / tracker->limit()) *
- 10000;
+ 10000);
max_pq.emplace(overcommit_ratio, tracker->label());
query_consumption[tracker->label()] = tracker->consumption();
}
diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp
index 1b0430d64fb..19ed39a6ab7 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -119,7 +119,7 @@ int64_t WorkloadGroup::memory_used() {
}
void WorkloadGroup::set_weighted_memory_used(int64_t wg_total_mem_used, double ratio) {
- _weighted_mem_used.store(wg_total_mem_used * ratio, std::memory_order_relaxed);
+ _weighted_mem_used.store(int64_t(wg_total_mem_used * ratio), std::memory_order_relaxed);
}
void WorkloadGroup::add_mem_tracker_limiter(std::shared_ptr<MemTrackerLimiter> mem_tracker_ptr) {
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp b/be/src/runtime/workload_group/workload_group_manager.cpp
index 027cfb2b2dd..78e2cd1fe06 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -198,7 +198,7 @@ void WorkloadGroupMgr::refresh_wg_memory_info() {
for (auto& wg : _workload_groups) {
auto wg_mem_limit = wg.second->memory_limit();
auto& wg_mem_info = wgs_mem_info[wg.first];
- wg_mem_info.weighted_mem_used = wg_mem_info.total_mem_used * ratio;
+ wg_mem_info.weighted_mem_used = int64_t(wg_mem_info.total_mem_used * ratio);
wg_mem_info.mem_used_ratio = (double)wg_mem_info.weighted_mem_used / wg_mem_limit;
wg.second->set_weighted_memory_used(wg_mem_info.total_mem_used, ratio);
@@ -237,7 +237,7 @@ void WorkloadGroupMgr::refresh_wg_memory_info() {
continue;
}
auto query_consumption = query_ctx->query_mem_tracker->consumption();
- int64_t query_weighted_consumption = query_consumption * ratio;
+ auto query_weighted_consumption = int64_t(query_consumption * ratio);
query_ctx->set_weighted_mem(query_weighted_mem_limit, query_weighted_consumption);
if (wg_mem_info.is_high_wartermark || wg_mem_info.is_low_wartermark) {
diff --git a/be/src/util/counts.h b/be/src/util/counts.h
index 8291d3a5b19..fec18cedcd6 100644
--- a/be/src/util/counts.h
+++ b/be/src/util/counts.h
@@ -83,8 +83,8 @@ public:
double get_percentile(std::vector<std::pair<int64_t, uint32_t>>& counts,
double position) const {
- long lower = std::floor(position);
- long higher = std::ceil(position);
+ long lower = long(std::floor(position));
+ long higher = long(std::ceil(position));
auto iter = counts.begin();
for (; iter != counts.end() && iter->second < lower + 1; ++iter)
diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp
index 6ead3a61ead..116dacb8da7 100644
--- a/be/src/util/cpu_info.cpp
+++ b/be/src/util/cpu_info.cpp
@@ -214,7 +214,7 @@ void CpuInfo::init() {
int num_cores = cgroup_bandwidth_quota(physical_num_cores);
if (max_mhz != 0) {
- cycles_per_ms_ = max_mhz * 1000;
+ cycles_per_ms_ = int64_t(max_mhz) * 1000;
} else {
cycles_per_ms_ = 1000000;
}
diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp
index 94b74b58280..4363cb32440 100644
--- a/be/src/util/date_func.cpp
+++ b/be/src/util/date_func.cpp
@@ -130,7 +130,7 @@ int32_t timev2_to_buffer_from_double(double time, char* buffer, int scale) {
time = -time;
*buffer++ = '-';
}
- int64_t m_time = time;
+ auto m_time = int64_t(time);
// m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second * 1000 * 1000 + microsecond
m_time = check_over_max_time(m_time);
int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000);
@@ -196,7 +196,7 @@ std::string timev2_to_buffer_from_double(double time, int scale) {
time = -time;
fmt::format_to(buffer, "-");
}
- int64_t m_time = time;
+ auto m_time = int64_t(time);
m_time = check_over_max_time(m_time);
// m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second * 1000 * 1000 + microsecond
int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000);
diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp
index 402797a8e35..8cb1bc2754a 100644
--- a/be/src/util/hash_util.hpp
+++ b/be/src/util/hash_util.hpp
@@ -41,7 +41,7 @@ public:
template <typename T>
static uint32_t fixed_len_to_uint32(T value) {
if constexpr (sizeof(T) <= sizeof(uint32_t)) {
- return value;
+ return (uint32_t)value;
}
return std::hash<T>()(value);
}
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index 17abbb8e93e..823490942af 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -354,10 +354,10 @@ int64_t MemInfo::tg_enable_overcommit_group_gc(int64_t request_free_memory,
}
// todo: GC according to resource group priority
- int64_t tg_need_free_memory =
+ auto tg_need_free_memory = int64_t(
gc_all_exceeded ? exceeded_memorys[i]
: static_cast<double>(exceeded_memorys[i]) / total_exceeded_memory *
- request_free_memory /* exceeded memory as a weight */;
+ request_free_memory); // exceeded memory as a weight
auto workload_group = task_groups[i];
total_free_memory += workload_group->gc_memory(tg_need_free_memory, profile);
}
@@ -423,7 +423,7 @@ void MemInfo::init() {
_s_mem_limit = _s_physical_mem;
}
_s_mem_limit_str = PrettyPrinter::print(_s_mem_limit, TUnit::BYTES);
- _s_soft_mem_limit = _s_mem_limit * config::soft_mem_limit_frac;
+ _s_soft_mem_limit = int64_t(_s_mem_limit * config::soft_mem_limit_frac);
_s_soft_mem_limit_str = PrettyPrinter::print(_s_soft_mem_limit, TUnit::BYTES);
_s_process_minor_gc_size =
@@ -456,9 +456,9 @@ void MemInfo::init() {
//
// upper sys_mem_available_low_water_mark, avoid wasting too much memory.
_s_sys_mem_available_low_water_mark = std::max<int64_t>(
- std::min<int64_t>(
- std::min<int64_t>(_s_physical_mem - _s_mem_limit, _s_physical_mem * 0.1),
- config::max_sys_mem_available_low_water_mark_bytes),
+ std::min<int64_t>(std::min<int64_t>(_s_physical_mem - _s_mem_limit,
+ int64_t(_s_physical_mem * 0.1)),
+ config::max_sys_mem_available_low_water_mark_bytes),
0);
_s_sys_mem_available_warning_water_mark = _s_sys_mem_available_low_water_mark * 2;
}
diff --git a/be/src/util/os_util.cpp b/be/src/util/os_util.cpp
index b98b9aa7d5a..13ee4032f67 100644
--- a/be/src/util/os_util.cpp
+++ b/be/src/util/os_util.cpp
@@ -83,13 +83,13 @@ Status parse_stat(const std::string& buffer, std::string* name, ThreadStats* sta
int64_t tmp;
if (safe_strto64(splits[kUserTicks], &tmp)) {
- stats->user_ns = tmp * (1e9 / kTicksPerSec);
+ stats->user_ns = int64_t(tmp * (1e9 / kTicksPerSec));
}
if (safe_strto64(splits[kKernelTicks], &tmp)) {
- stats->kernel_ns = tmp * (1e9 / kTicksPerSec);
+ stats->kernel_ns = int64_t(tmp * (1e9 / kTicksPerSec));
}
if (safe_strto64(splits[kIoWait], &tmp)) {
- stats->iowait_ns = tmp * (1e9 / kTicksPerSec);
+ stats->iowait_ns = int64_t(tmp * (1e9 / kTicksPerSec));
}
if (name != nullptr) {
*name = extracted_name;
diff --git a/be/src/util/parse_util.cpp b/be/src/util/parse_util.cpp
index 4e685715f61..8e810764e7b 100644
--- a/be/src/util/parse_util.cpp
+++ b/be/src/util/parse_util.cpp
@@ -82,12 +82,12 @@ int64_t ParseUtil::parse_mem_spec(const std::string& mem_spec_str, int64_t paren
}
if (multiplier != -1) {
- bytes = multiplier * limit_val;
+ bytes = int64_t(multiplier * limit_val);
} else if (*is_percent) {
if (parent_limit == -1) {
- bytes = (static_cast<double>(limit_val) / 100.0) * physical_mem;
+ bytes = int64_t(static_cast<double>(limit_val) / 100.0 * physical_mem);
} else {
- bytes = (static_cast<double>(limit_val) / 100.0) * parent_limit;
+ bytes = int64_t(static_cast<double>(limit_val) / 100.0 * parent_limit);
}
}
} else {
diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp
index f5ed7250618..647c6e966c5 100644
--- a/be/src/util/runtime_profile.cpp
+++ b/be/src/util/runtime_profile.cpp
@@ -617,7 +617,7 @@ int64_t RuntimeProfile::units_per_second(const RuntimeProfile::Counter* total_co
}
double secs = static_cast<double>(timer->value()) / 1000.0 / 1000.0 / 1000.0;
- return total_counter->value() / secs;
+ return int64_t(total_counter->value() / secs);
}
int64_t RuntimeProfile::counter_sum(const std::vector<Counter*>* counters) {
diff --git a/be/src/util/url_coding.cpp b/be/src/util/url_coding.cpp
index 6ddd4c05401..d0bbf5aae63 100644
--- a/be/src/util/url_coding.cpp
+++ b/be/src/util/url_coding.cpp
@@ -88,7 +88,7 @@ bool url_decode(const std::string& in, std::string* out) {
}
void base64_encode(const std::string& in, std::string* out) {
- out->resize(in.length() * (4.0 / 3) + 1);
+ out->resize(size_t(in.length() * (4.0 / 3) + 1));
auto len = base64_encode(reinterpret_cast<const unsigned char*>(in.c_str()), in.length(),
(unsigned char*)out->c_str());
out->resize(len);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
index 498ee20ccb8..e8596a00546 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h
@@ -85,7 +85,7 @@ struct AggregateFunctionAvgWeightedData {
weight_sum = 0.0;
}
- double get() const { return weight_sum ? data_sum / weight_sum : std::nan(""); }
+ double get() const { return data_sum / weight_sum; }
double data_sum = 0.0;
double weight_sum = 0.0;
diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.h b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
index 456e91c3f6a..f095afb41b1 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_stddev.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
@@ -104,7 +104,7 @@ struct BaseData {
double sum_count = count + rhs.count;
mean = rhs.mean + delta * count / sum_count;
m2 = rhs.m2 + m2 + (delta * delta) * rhs.count * count / sum_count;
- count = sum_count;
+ count = int64_t(sum_count);
}
void add(const IColumn* column, size_t row_num) {
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index afb97542e5a..f1bcee1ad7d 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -496,8 +496,6 @@ public:
*/
using ColumnIndex = UInt64;
using Selector = PaddedPODArray<ColumnIndex>;
- virtual std::vector<MutablePtr> scatter(ColumnIndex num_columns,
- const Selector& selector) const = 0;
virtual void append_data_by_selector(MutablePtr& res, const Selector& selector) const = 0;
@@ -695,11 +693,6 @@ public:
bool is_date_time = false;
protected:
- /// Template is to devirtualize calls to insert_from method.
- /// In derived classes (that use final keyword), implement scatter method as call to scatter_impl.
- template <typename Derived>
- std::vector<MutablePtr> scatter_impl(ColumnIndex num_columns, const Selector& selector) const;
-
template <typename Derived>
void append_data_by_selector_impl(MutablePtr& res, const Selector& selector) const;
};
diff --git a/be/src/vec/columns/column_array.h b/be/src/vec/columns/column_array.h
index 5f6db27c594..3176f7a45c6 100644
--- a/be/src/vec/columns/column_array.h
+++ b/be/src/vec/columns/column_array.h
@@ -199,10 +199,6 @@ public:
const ColumnPtr& get_offsets_ptr() const { return offsets; }
ColumnPtr& get_offsets_ptr() { return offsets; }
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
- return scatter_impl<ColumnArray>(num_columns, selector);
- }
-
size_t ALWAYS_INLINE offset_at(ssize_t i) const { return get_offsets()[i - 1]; }
size_t ALWAYS_INLINE size_at(ssize_t i) const {
return get_offsets()[i] - get_offsets()[i - 1];
diff --git a/be/src/vec/columns/column_complex.h b/be/src/vec/columns/column_complex.h
index 70d32da0b9b..d983fc9175e 100644
--- a/be/src/vec/columns/column_complex.h
+++ b/be/src/vec/columns/column_complex.h
@@ -258,12 +258,6 @@ public:
ColumnPtr replicate(const IColumn::Offsets& replicate_offsets) const override;
- [[noreturn]] MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- LOG(FATAL) << "scatter not implemented";
- __builtin_unreachable();
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
this->template append_data_by_selector_impl<ColumnComplexType<T>>(res, selector);
diff --git a/be/src/vec/columns/column_const.cpp b/be/src/vec/columns/column_const.cpp
index e06e53b4289..096ec91ecea 100644
--- a/be/src/vec/columns/column_const.cpp
+++ b/be/src/vec/columns/column_const.cpp
@@ -126,22 +126,6 @@ void ColumnConst::update_hashes_with_value(uint64_t* __restrict hashes,
}
}
-MutableColumns ColumnConst::scatter(ColumnIndex num_columns, const Selector& selector) const {
- if (s != selector.size()) {
- LOG(FATAL) << fmt::format("Size of selector ({}) doesn't match size of column ({})",
- selector.size(), s);
- }
-
- std::vector<size_t> counts = count_columns_size_in_selector(num_columns, selector);
-
- MutableColumns res(num_columns);
- for (size_t i = 0; i < num_columns; ++i) {
- res[i] = clone_resized(counts[i]);
- }
-
- return res;
-}
-
void ColumnConst::get_permutation(bool /*reverse*/, size_t /*limit*/, int /*nan_direction_hint*/,
Permutation& res) const {
res.resize(s);
diff --git a/be/src/vec/columns/column_const.h b/be/src/vec/columns/column_const.h
index bd6a817c283..016e4257402 100644
--- a/be/src/vec/columns/column_const.h
+++ b/be/src/vec/columns/column_const.h
@@ -220,8 +220,6 @@ public:
}
}
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override;
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
assert_cast<Self&>(*res).resize(selector.size());
diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index 0dbd8c14027..152d4165416 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -227,11 +227,6 @@ public:
ColumnPtr replicate(const IColumn::Offsets& offsets) const override;
- MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- return this->template scatter_impl<Self>(num_columns, selector);
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
this->template append_data_by_selector_impl<Self>(res, selector);
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index 518d8faa008..175912f9668 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -190,12 +190,6 @@ public:
__builtin_unreachable();
}
- [[noreturn]] MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- LOG(FATAL) << "scatter not supported in ColumnDictionary";
- __builtin_unreachable();
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
LOG(FATAL) << "append_data_by_selector is not supported in ColumnDictionary!";
diff --git a/be/src/vec/columns/column_dummy.h b/be/src/vec/columns/column_dummy.h
index b94464be5ba..f330a60642a 100644
--- a/be/src/vec/columns/column_dummy.h
+++ b/be/src/vec/columns/column_dummy.h
@@ -116,20 +116,6 @@ public:
return clone_dummy(offsets.back());
}
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
- if (s != selector.size()) {
- LOG(FATAL) << "Size of selector doesn't match size of column.";
- }
-
- std::vector<size_t> counts(num_columns);
- for (auto idx : selector) ++counts[idx];
-
- MutableColumns res(num_columns);
- for (size_t i = 0; i < num_columns; ++i) res[i] = clone_resized(counts[i]);
-
- return res;
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
size_t num_rows = size();
diff --git a/be/src/vec/columns/column_fixed_length_object.h b/be/src/vec/columns/column_fixed_length_object.h
index d44ebce53a7..5875b8f6c30 100644
--- a/be/src/vec/columns/column_fixed_length_object.h
+++ b/be/src/vec/columns/column_fixed_length_object.h
@@ -233,12 +233,6 @@ public:
return res;
}
- [[noreturn]] MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- LOG(FATAL) << "scatter not supported";
- __builtin_unreachable();
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
this->template append_data_by_selector_impl<Self>(res, selector);
diff --git a/be/src/vec/columns/column_impl.h b/be/src/vec/columns/column_impl.h
index 53f6be6c8c8..20292ad2351 100644
--- a/be/src/vec/columns/column_impl.h
+++ b/be/src/vec/columns/column_impl.h
@@ -32,33 +32,6 @@
namespace doris::vectorized {
-template <typename Derived>
-std::vector<IColumn::MutablePtr> IColumn::scatter_impl(ColumnIndex num_columns,
- const Selector& selector) const {
- size_t num_rows = size();
-
- if (num_rows != selector.size()) {
- LOG(FATAL) << fmt::format("Size of selector: {}, doesn't match size of column:{}",
- selector.size(), num_rows);
- }
-
- std::vector<MutablePtr> columns(num_columns);
- for (auto& column : columns) column = clone_empty();
-
- {
- size_t reserve_size =
- num_rows * 1.1 / num_columns; /// 1.1 is just a guess. Better to use n-sigma rule.
-
- if (reserve_size > 1)
- for (auto& column : columns) column->reserve(reserve_size);
- }
-
- for (size_t i = 0; i < num_rows; ++i)
- static_cast<Derived&>(*columns[selector[i]]).insert_from(*this, i);
-
- return columns;
-}
-
template <typename Derived>
void IColumn::append_data_by_selector_impl(MutablePtr& res, const Selector& selector) const {
size_t num_rows = size();
diff --git a/be/src/vec/columns/column_map.h b/be/src/vec/columns/column_map.h
index 0fe61ccdaeb..ed5a692defd 100644
--- a/be/src/vec/columns/column_map.h
+++ b/be/src/vec/columns/column_map.h
@@ -118,9 +118,6 @@ public:
size_t filter(const Filter& filter) override;
ColumnPtr permute(const Permutation& perm, size_t limit) const override;
ColumnPtr replicate(const Offsets& offsets) const override;
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
- return scatter_impl<ColumnMap>(num_columns, selector);
- }
int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override;
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index 1bec1165aae..462b28d3a7d 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -227,10 +227,6 @@ public:
void update_hashes_with_value(uint64_t* __restrict hashes,
const uint8_t* __restrict null_data) const override;
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
- return scatter_impl<ColumnNullable>(num_columns, selector);
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
append_data_by_selector_impl<ColumnNullable>(res, selector);
diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
index 39d156d681b..a940b4a8811 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -456,11 +456,6 @@ public:
LOG(FATAL) << "should not call the method in column object";
}
- MutableColumns scatter(ColumnIndex, const Selector&) const override {
- LOG(FATAL) << "should not call the method in column object";
- return {};
- }
-
void replace_column_data(const IColumn&, size_t row, size_t self_row) override {
LOG(FATAL) << "should not call the method in column object";
}
diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h
index 04a599f5e9b..b858ab86bf3 100644
--- a/be/src/vec/columns/column_string.h
+++ b/be/src/vec/columns/column_string.h
@@ -516,10 +516,6 @@ public:
ColumnPtr replicate(const Offsets& replicate_offsets) const override;
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
- return scatter_impl<ColumnString>(num_columns, selector);
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
append_data_by_selector_impl<ColumnString>(res, selector);
diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp
index b1a415de272..d075b040e6d 100644
--- a/be/src/vec/columns/column_struct.cpp
+++ b/be/src/vec/columns/column_struct.cpp
@@ -319,29 +319,6 @@ MutableColumnPtr ColumnStruct::get_shrinked_column() {
return ColumnStruct::create(std::move(new_columns));
}
-MutableColumns ColumnStruct::scatter(ColumnIndex num_columns, const Selector& selector) const {
- const size_t tuple_size = columns.size();
- std::vector<MutableColumns> scattered_tuple_elements(tuple_size);
-
- for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx) {
- scattered_tuple_elements[tuple_element_idx] =
- columns[tuple_element_idx]->scatter(num_columns, selector);
- }
-
- MutableColumns res(num_columns);
-
- for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx) {
- MutableColumns new_columns(tuple_size);
- for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx) {
- new_columns[tuple_element_idx] =
- std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
- }
- res[scattered_idx] = ColumnStruct::create(std::move(new_columns));
- }
-
- return res;
-}
-
void ColumnStruct::reserve(size_t n) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
index ce4b5f461d7..34d1e3ecf0e 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -151,7 +151,6 @@ public:
size_t filter(const Filter& filter) override;
ColumnPtr permute(const Permutation& perm, size_t limit) const override;
ColumnPtr replicate(const Offsets& offsets) const override;
- MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override;
// ColumnPtr index(const IColumn & indexes, size_t limit) const override;
int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override;
diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h
index a2cfb7cd041..7e035b8b47a 100644
--- a/be/src/vec/columns/column_vector.h
+++ b/be/src/vec/columns/column_vector.h
@@ -191,10 +191,14 @@ public:
}
void insert_range_of_integer(T begin, T end) {
- auto old_size = data.size();
- data.resize(old_size + (end - begin));
- for (int i = 0; i < end - begin; i++) {
- data[old_size + i] = begin + i;
+ if constexpr (std::is_integral_v<T>) {
+ auto old_size = data.size();
+ data.resize(old_size + (end - begin));
+ for (int i = 0; i < end - begin; i++) {
+ data[old_size + i] = begin + i;
+ }
+ } else {
+ LOG(FATAL) << "double column not support insert_range_of_integer";
}
}
@@ -404,11 +408,6 @@ public:
ColumnPtr replicate(const IColumn::Offsets& offsets) const override;
- MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- return this->template scatter_impl<Self>(num_columns, selector);
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
this->template append_data_by_selector_impl<Self>(res, selector);
diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h
index 198c7ee9cd4..2a390d39d29 100644
--- a/be/src/vec/columns/predicate_column.h
+++ b/be/src/vec/columns/predicate_column.h
@@ -429,12 +429,6 @@ public:
__builtin_unreachable();
}
- [[noreturn]] MutableColumns scatter(IColumn::ColumnIndex num_columns,
- const IColumn::Selector& selector) const override {
- LOG(FATAL) << "scatter not supported in PredicateColumnType";
- __builtin_unreachable();
- }
-
void append_data_by_selector(MutableColumnPtr& res,
const IColumn::Selector& selector) const override {
LOG(FATAL) << "append_data_by_selector is not supported in PredicateColumnType!";
diff --git a/be/src/vec/common/uint128.h b/be/src/vec/common/uint128.h
index a9c30328ab7..28b1f62dc72 100644
--- a/be/src/vec/common/uint128.h
+++ b/be/src/vec/common/uint128.h
@@ -20,6 +20,7 @@
#pragma once
+#include <cstdint>
#include <iomanip>
#include <sstream>
#include <tuple>
@@ -39,7 +40,13 @@ struct UInt128 {
UInt128() = default;
explicit UInt128(const UInt64 low_, const UInt64 high_) : low(low_), high(high_) {}
+ explicit UInt128(const Int128 rhs) : low(rhs % UINT64_MAX), high(rhs / UINT64_MAX) {}
explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
+ explicit UInt128(const int rhs) : low(rhs), high() {}
+ explicit UInt128(const int64_t rhs) : low(rhs), high() {}
+ explicit UInt128(const uint32_t rhs) : low(rhs), high() {}
+ explicit UInt128(const double rhs)
+ : low((UInt64)rhs % UINT64_MAX), high(UInt64(rhs / double(INT64_MAX))) {}
auto tuple() const { return std::tie(high, low); }
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index fd3e16e0a2b..73025b166e5 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -541,11 +541,11 @@ struct Decimal {
/// If T is integral, the given value will be rounded to integer.
template <std::floating_point U>
- static constexpr U type_round(U value) noexcept {
+ static constexpr T type_round(U value) noexcept {
if constexpr (wide::IntegralConcept<T>()) {
- return round(value);
+ return T(round(value));
}
- return value;
+ return T(value);
}
static Decimal double_to_decimal(double value_) {
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index bc53fac6a37..e708cda164e 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -57,7 +57,7 @@ using DataTypePtr = std::shared_ptr<const IDataType>;
using DataTypes = std::vector<DataTypePtr>;
constexpr auto SERIALIZED_MEM_SIZE_LIMIT = 256;
inline size_t upper_int32(size_t size) {
- return (3 + size) / 4.0;
+ return size_t((3 + size) / 4.0);
}
/** Properties of data type.
diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
index 234078ad4c1..8bd7f8b0153 100644
--- a/be/src/vec/data_types/data_type_decimal.h
+++ b/be/src/vec/data_types/data_type_decimal.h
@@ -653,7 +653,8 @@ void convert_to_decimal(typename ToDataType::FieldType* dst,
}
}
for (size_t i = 0; i < size; ++i) {
- dst[i].value = FromFieldType(src[i] * multiplier.value + ((src[i] >= 0) ? 0.5 : -0.5));
+ dst[i].value = typename ToDataType::FieldType::NativeType(
+ FromFieldType(src[i] * multiplier.value + ((src[i] >= 0) ? 0.5 : -0.5)));
}
} else {
using DecimalFrom =
diff --git a/be/src/vec/data_types/serde/data_type_number_serde.h b/be/src/vec/data_types/serde/data_type_number_serde.h
index fedee87d3bd..28e2ade1664 100644
--- a/be/src/vec/data_types/serde/data_type_number_serde.h
+++ b/be/src/vec/data_types/serde/data_type_number_serde.h
@@ -330,7 +330,7 @@ Status DataTypeNumberSerDe<T>::read_one_cell_from_json(IColumn& column,
} else if (value.IsInt64()) {
col.insert_value((T)value.GetInt64());
} else if (value.IsFloat() || value.IsDouble()) {
- col.insert_value((T)value.GetDouble());
+ col.insert_value(T(value.GetDouble()));
} else {
CHECK(false) << "Improssible";
}
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index bb04046ef9f..8d4a9b3a164 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -327,8 +327,8 @@ void ScannerContext::_try_to_scale_up() {
bool is_scale_up = false;
// calculate the number of scanners that can be scheduled
- int num_add = std::min(_num_running_scanners * SCALE_UP_RATIO,
- _max_thread_num * MAX_SCALE_UP_RATIO - _num_running_scanners);
+ int num_add = int(std::min(_num_running_scanners * SCALE_UP_RATIO,
+ _max_thread_num * MAX_SCALE_UP_RATIO - _num_running_scanners));
if (_estimated_block_size > 0) {
int most_add =
(_max_bytes_in_queue - _free_blocks_memory_usage) / _estimated_block_size;
diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp
index 16c267b26df..d1f4d8ac2c8 100644
--- a/be/src/vec/exec/vset_operation_node.cpp
+++ b/be/src/vec/exec/vset_operation_node.cpp
@@ -521,8 +521,8 @@ void VSetOperationNode<is_intersect>::refresh_hash_table() {
bool is_need_shrink =
arg.hash_table->should_be_shrink(_valid_element_in_hash_tbl);
if (is_intersect || is_need_shrink) {
- tmp_hash_table->init_buf_size(
- _valid_element_in_hash_tbl / arg.hash_table->get_factor() + 1);
+ tmp_hash_table->init_buf_size(size_t(
+ _valid_element_in_hash_tbl / arg.hash_table->get_factor() + 1));
}
arg.init_iterator();
diff --git a/be/src/vec/exprs/table_function/vexplode_json_array.cpp b/be/src/vec/exprs/table_function/vexplode_json_array.cpp
index 1a149d949af..fbff29390e9 100644
--- a/be/src/vec/exprs/table_function/vexplode_json_array.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_json_array.cpp
@@ -66,7 +66,7 @@ int ParsedData::set_output(ExplodeJsonArrayType type, rapidjson::Document& docum
} else if (value < min_value) {
_backup_int[i] = min_value;
} else {
- _backup_int[i] = value;
+ _backup_int[i] = long(value);
}
_data[i] = &_backup_int[i];
} else {
diff --git a/be/src/vec/functions/array/function_array_join.h b/be/src/vec/functions/array/function_array_join.h
index 583738e4319..d184765edf6 100644
--- a/be/src/vec/functions/array/function_array_join.h
+++ b/be/src/vec/functions/array/function_array_join.h
@@ -147,7 +147,7 @@ private:
if (is_decimal) {
DecimalV2Value decimal_value =
- (DecimalV2Value)(src_data_concrete->get_data()[j]);
+ (DecimalV2Value)(int128_t(src_data_concrete->get_data()[j]));
std::string decimal_str = decimal_value.to_string();
_fill_result_string(decimal_str, sep_str, result_str, is_first_elem);
} else {
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 79762af86ce..22022925f8d 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -216,12 +216,12 @@ struct TimeCast {
template <typename T, typename S>
//requires {std::is_arithmetic_v<T> && std::is_arithmetic_v<S>}
static bool try_parse_time(T from, S& x, const cctz::time_zone& local_time_zone) {
- int64 seconds = from / 100;
+ int64 seconds = int64(from / 100);
int64 hour = 0, minute = 0, second = 0;
- second = from - 100 * seconds;
+ second = int64(from - 100 * seconds);
from /= 100;
- seconds = from / 100;
- minute = from - 100 * seconds;
+ seconds = int64(from / 100);
+ minute = int64(from - 100 * seconds);
hour = seconds;
if (minute >= 60 || second >= 60) {
return false;
@@ -388,7 +388,7 @@ struct ConvertImpl {
DataTypeDateTimeV2::cast_to_date_v2(vec_from[i], vec_to[i]);
} else {
UInt32 scale = additions;
- vec_to[i] = vec_from[i] / std::pow(10, 6 - scale);
+ vec_to[i] = ToFieldType(vec_from[i] / std::pow(10, 6 - scale));
}
} else if constexpr (IsTimeType<ToDataType>) {
if constexpr (IsDateTimeType<ToDataType> && IsDateV2Type<FromDataType>) {
@@ -515,7 +515,7 @@ struct ConvertImplToTimeType {
[&](auto narrow_integral) {
for (size_t i = 0; i < size; ++i) {
auto& date_value = reinterpret_cast<DateValueType&>(vec_to[i]);
- vec_null_map_to[i] = !date_value.from_date_int64(vec_from[i]);
+ vec_null_map_to[i] = !date_value.from_date_int64(int64_t(vec_from[i]));
// DateType of VecDateTimeValue should cast to date
if constexpr (IsDateType<ToDataType>) {
date_value.cast_to_date();
diff --git a/be/src/vec/functions/function_quantile_state.cpp b/be/src/vec/functions/function_quantile_state.cpp
index 1d9afc209a3..524909314dc 100644
--- a/be/src/vec/functions/function_quantile_state.cpp
+++ b/be/src/vec/functions/function_quantile_state.cpp
@@ -134,7 +134,7 @@ public:
float compression = 2048;
if (compression_arg) {
auto compression_arg_val = compression_arg->get_value<Float32>();
- if (compression_arg_val && compression_arg_val >= QUANTILE_STATE_COMPRESSION_MIN &&
+ if (compression_arg_val >= QUANTILE_STATE_COMPRESSION_MIN &&
compression_arg_val <= QUANTILE_STATE_COMPRESSION_MAX) {
compression = compression_arg_val;
}
diff --git a/be/src/vec/functions/int_div.h b/be/src/vec/functions/int_div.h
index 605d6d8339a..91952a3db25 100644
--- a/be/src/vec/functions/int_div.h
+++ b/be/src/vec/functions/int_div.h
@@ -45,13 +45,13 @@ struct DivideIntegralImpl {
if (!is_null) {
if constexpr (!std::is_floating_point_v<A> && !std::is_same_v<A, Int128> &&
!std::is_same_v<A, Int8> && !std::is_same_v<A, UInt8>) {
- const auto divider = libdivide::divider<A>(b);
+ const auto divider = libdivide::divider<A>(A(b));
for (size_t i = 0; i < size; i++) {
c[i] = a[i] / divider;
}
} else {
for (size_t i = 0; i < size; i++) {
- c[i] = a[i] / b;
+ c[i] = Result(a[i] / b);
}
}
}
@@ -60,7 +60,7 @@ struct DivideIntegralImpl {
template <typename Result = ResultType>
static inline Result apply(A a, B b, UInt8& is_null) {
is_null = b == 0;
- return a / (b + is_null);
+ return Result(a / (b + is_null));
}
};
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index ce53029731a..cf6dc0b70ed 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -980,10 +980,10 @@ void VTabletWriter::_send_batch_process() {
SCOPED_ATTACH_TASK(_state);
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker);
- int sleep_time = config::olap_table_sink_send_interval_microseconds *
- (_vpartition->is_auto_partition()
- ? config::olap_table_sink_send_interval_auto_partition_factor
- : 1);
+ int sleep_time = int(config::olap_table_sink_send_interval_microseconds *
+ (_vpartition->is_auto_partition()
+ ? config::olap_table_sink_send_interval_auto_partition_factor
+ : 1));
while (true) {
// incremental open will temporarily make channels into abnormal state. stop checking when this.
diff --git a/be/test/olap/wal/wal_manager_test.cpp b/be/test/olap/wal/wal_manager_test.cpp
index 2dc59cec9d9..afb4c7696ef 100644
--- a/be/test/olap/wal/wal_manager_test.cpp
+++ b/be/test/olap/wal/wal_manager_test.cpp
@@ -171,13 +171,13 @@ TEST_F(WalManagerTest, TestDynamicWalSpaceLimt) {
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "5%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
- wal_limit_bytes = available_bytes * 0.05;
+ wal_limit_bytes = size_t(available_bytes * 0.05);
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "50%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
- wal_limit_bytes = available_bytes * 0.5;
+ wal_limit_bytes = size_t(available_bytes * 0.5);
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
diff --git a/be/test/vec/core/block_spill_test.cpp b/be/test/vec/core/block_spill_test.cpp
index 45156dee48c..af30479e10a 100644
--- a/be/test/vec/core/block_spill_test.cpp
+++ b/be/test/vec/core/block_spill_test.cpp
@@ -336,7 +336,7 @@ TEST_F(TestBlockSpill, TestDecimal) {
decimal_column.get())
->get_data();
for (int i = 0; i < total_rows; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
decimal_data.push_back(value);
}
vectorized::ColumnWithTypeAndName test_decimal(decimal_column->get_ptr(), decimal_data_type,
@@ -365,7 +365,7 @@ TEST_F(TestBlockSpill, TestDecimal) {
auto* real_column =
(vectorized::ColumnDecimal<vectorized::Decimal<vectorized::Int128>>*)column.get();
for (size_t j = 0; j < batch_size; ++j) {
- __int128_t value = (j + i * batch_size) * (pow(10, 9) + pow(10, 8));
+ __int128_t value = __int128_t((j + i * batch_size) * (pow(10, 9) + pow(10, 8)));
EXPECT_EQ(real_column->get_element(j).value, value);
}
}
@@ -392,7 +392,7 @@ TEST_F(TestBlockSpill, TestDecimalNullable) {
if ((i + 1) % batch_size == 0) {
nullable_col->insert_data(nullptr, 0);
} else {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
nullable_col->insert_data((const char*)&value, sizeof(value));
}
}
@@ -427,7 +427,7 @@ TEST_F(TestBlockSpill, TestDecimalNullable) {
if ((j + 1) % batch_size == 0) {
ASSERT_TRUE(real_column->is_null_at(j));
} else {
- __int128_t value = (j + i * batch_size) * (pow(10, 9) + pow(10, 8));
+ __int128_t value = __int128_t((j + i * batch_size) * (pow(10, 9) + pow(10, 8)));
EXPECT_EQ(decimal_col.get_element(j).value, value);
}
}
diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp
index 4b2ae60fdf7..db770ef29da 100644
--- a/be/test/vec/core/block_test.cpp
+++ b/be/test/vec/core/block_test.cpp
@@ -165,7 +165,7 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
vectorized::ColumnWithTypeAndName type_and_name(decimal_column->get_ptr(),
@@ -322,7 +322,7 @@ TEST(BlockTest, dump_data) {
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
decimal_data.push_back(value);
}
vectorized::ColumnWithTypeAndName test_decimal(decimal_column->get_ptr(), decimal_data_type,
diff --git a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
index bce3e419775..4793ae8128a 100644
--- a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
@@ -244,7 +244,7 @@ void serialize_and_deserialize_arrow_test() {
decimal_column.get())
->get_data();
for (int i = 0; i < row_num; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
vectorized::ColumnWithTypeAndName type_and_name(decimal_column->get_ptr(),
diff --git a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
index 420687329bd..35c169da7aa 100644
--- a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
@@ -158,7 +158,7 @@ void serialize_and_deserialize_mysql_test() {
decimal_column.get())
->get_data();
for (int i = 0; i < row_num; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ auto value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
vectorized::ColumnWithTypeAndName type_and_name(decimal_column->get_ptr(),
diff --git a/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp b/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
index 78cd4649998..beda6c88a9c 100644
--- a/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
@@ -109,7 +109,7 @@ inline void serialize_and_deserialize_pb_test() {
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
check_pb_col(decimal_data_type, *decimal_column.get());
diff --git a/be/test/vec/data_types/serde/data_type_serde_test.cpp b/be/test/vec/data_types/serde/data_type_serde_test.cpp
index ea1626790a3..ef235254db5 100644
--- a/be/test/vec/data_types/serde/data_type_serde_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_test.cpp
@@ -107,7 +107,7 @@ inline void serialize_and_deserialize_pb_test() {
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
check_pb_col(decimal_data_type, *decimal_column.get());
diff --git a/be/test/vec/jsonb/serialize_test.cpp b/be/test/vec/jsonb/serialize_test.cpp
index bfbf5620e08..04eb96dce62 100644
--- a/be/test/vec/jsonb/serialize_test.cpp
+++ b/be/test/vec/jsonb/serialize_test.cpp
@@ -385,7 +385,7 @@ TEST(BlockSerializeTest, JsonbBlock) {
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
- __int128_t value = i * pow(10, 9) + i * pow(10, 8);
+ __int128_t value = __int128_t(i * pow(10, 9) + i * pow(10, 8));
data.push_back(value);
}
vectorized::ColumnWithTypeAndName type_and_name(decimal_column->get_ptr(),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/13: [fix](inverted index) fix the incorrect result issue of COUNT_ON_INDEX for key columns (#33164)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9670422d61e98ff61eeaa4202c8d8244899571a4
Author: zzzxl <33...@users.noreply.github.com>
AuthorDate: Wed Apr 3 15:44:56 2024 +0800
[fix](inverted index) fix the incorrect result issue of COUNT_ON_INDEX for key columns (#33164)
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 20 +++++--
be/src/olap/rowset/segment_v2/segment_iterator.h | 5 +-
.../data/inverted_index_p0/test_index_delete.out | 16 ++++++
.../inverted_index_p0/test_index_delete.groovy | 65 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index e4a139d9c3b..e69d707499e 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1093,9 +1093,7 @@ bool SegmentIterator::_need_read_data(ColumnId cid) {
return true;
}
// if there is delete predicate, we always need to read data
- std::set<uint32_t> delete_columns_set;
- _opts.delete_condition_predicates->get_all_column_ids(delete_columns_set);
- if (delete_columns_set.count(cid) > 0) {
+ if (_has_delete_predicate(cid)) {
return true;
}
if (_output_columns.count(-1)) {
@@ -1869,7 +1867,7 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32
for (auto cid : _first_read_column_ids) {
auto& column = _current_return_columns[cid];
- if (_need_read_key_data(cid, column, nrows_read)) {
+ if (_no_need_read_key_data(cid, column, nrows_read)) {
continue;
}
if (_prune_column(cid, column, true, nrows_read)) {
@@ -2631,8 +2629,8 @@ void SegmentIterator::_calculate_pred_in_remaining_conjunct_root(
}
}
-bool SegmentIterator::_need_read_key_data(ColumnId cid, vectorized::MutableColumnPtr& column,
- size_t nrows_read) {
+bool SegmentIterator::_no_need_read_key_data(ColumnId cid, vectorized::MutableColumnPtr& column,
+ size_t nrows_read) {
if (_opts.tablet_schema->keys_type() != KeysType::DUP_KEYS) {
return false;
}
@@ -2645,6 +2643,10 @@ bool SegmentIterator::_need_read_key_data(ColumnId cid, vectorized::MutableColum
return false;
}
+ if (_has_delete_predicate(cid)) {
+ return false;
+ }
+
std::set<uint32_t> cids;
for (auto* pred : _col_predicates) {
cids.insert(pred->column_id());
@@ -2669,5 +2671,11 @@ bool SegmentIterator::_need_read_key_data(ColumnId cid, vectorized::MutableColum
return true;
}
+bool SegmentIterator::_has_delete_predicate(ColumnId cid) {
+ std::set<uint32_t> delete_columns_set;
+ _opts.delete_condition_predicates->get_all_column_ids(delete_columns_set);
+ return delete_columns_set.contains(cid);
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index e6830e84c90..1bf7cf22c3d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -375,7 +375,10 @@ private:
Status _convert_to_expected_type(const std::vector<ColumnId>& col_ids);
- bool _need_read_key_data(ColumnId cid, vectorized::MutableColumnPtr& column, size_t nrows_read);
+ bool _no_need_read_key_data(ColumnId cid, vectorized::MutableColumnPtr& column,
+ size_t nrows_read);
+
+ bool _has_delete_predicate(ColumnId cid);
class BitmapRangeIterator;
class BackwardBitmapRangeIterator;
diff --git a/regression-test/data/inverted_index_p0/test_index_delete.out b/regression-test/data/inverted_index_p0/test_index_delete.out
new file mode 100644
index 00000000000..e0f6bfb1311
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_index_delete.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+8
+
+-- !sql --
+8
+
+-- !sql --
+8
+
+-- !sql --
+6
+
+-- !sql --
+0
+
diff --git a/regression-test/suites/inverted_index_p0/test_index_delete.groovy b/regression-test/suites/inverted_index_p0/test_index_delete.groovy
new file mode 100644
index 00000000000..a189884508b
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_index_delete.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_index_delete", "p0") {
+ def indexTbName1 = "test_index_delete"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+
+ sql """
+ CREATE TABLE ${indexTbName1} (
+ `a` int(11) NULL COMMENT "",
+ `b` text NULL COMMENT "",
+ INDEX b_idx (`b`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`a`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ sql """ INSERT INTO ${indexTbName1} VALUES (1, "1"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (2, "1"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (3, "1"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (4, "2"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (5, "2"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (6, "2"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (7, "3"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (8, "3"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (9, "3"); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (10, "4"); """
+
+ try {
+ sql "sync"
+
+ sql """ delete from ${indexTbName1} where a >= 9; """
+ qt_sql """ select count() from ${indexTbName1} where a >= 1 and a <= 10; """
+ qt_sql """ select count() from ${indexTbName1} where a >= 1; """
+ qt_sql """ select count() from ${indexTbName1} where a <= 10; """
+
+ sql """ delete from ${indexTbName1} where b = '3'; """
+ qt_sql """ select count() from ${indexTbName1} where a >= 1; """
+ qt_sql """ select count() from ${indexTbName1} where b match '3'; """
+
+ } finally {
+ //try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/13: [Fix](nereids) modify the binding aggregate function in order by (#32758)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5e59c09a602797820bd78872c690d9be9fd60c70
Author: feiniaofeiafei <53...@users.noreply.github.com>
AuthorDate: Wed Apr 3 15:32:07 2024 +0800
[Fix](nereids) modify the binding aggregate function in order by (#32758)
modify the bind logical to make the order by has same behavior with mysql when sort child is aggregate.
when an order by Expr has aggregate function, all slots in this order by Expr should bind the LogicalAggregate non-AggFunction outputs first, then bind the LogicalAggregate Child
e.g.
select 2*abs(sum(c1)) as c1, c1,sum(c1)+c1 from t_order_by_bind_priority group by c1 order by sum(c1)+c1 asc;
in this sql, the two c1 in order by all bind to the c1 in t_order_by_bind_priority
---
.../nereids/rules/analysis/BindExpression.java | 57 +++++++++++++++++++++-
.../nereids/rules/analysis/FillUpMissingSlots.java | 2 +-
.../rules/analysis/FillUpMissingSlotsTest.java | 2 +-
.../nereids_syntax_p0/order_by_bind_priority.out | 25 ++++++++++
.../order_by_bind_priority.groovy | 38 +++++++++++++++
5 files changed, 120 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 43f89d5b010..8c8a447e830 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -759,7 +759,7 @@ public class BindExpression implements AnalysisRuleFactory {
final Plan finalInput = input;
Supplier<Scope> inputChildrenScope = Suppliers.memoize(
() -> toScope(cascadesContext, PlanUtils.fastGetChildrenOutputs(finalInput.children())));
- SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer(
+ SimpleExprAnalyzer bindInInputScopeThenInputChildScope = buildCustomSlotBinderAnalyzer(
sort, cascadesContext, inputScope, true, false,
(self, unboundSlot) -> {
// first, try to bind slot in Scope(input.output)
@@ -774,9 +774,17 @@ public class BindExpression implements AnalysisRuleFactory {
return self.bindExactSlotsByThisScope(unboundSlot, inputChildrenScope.get());
});
+ SimpleExprAnalyzer bindInInputChildScope = getAnalyzerForOrderByAggFunc(finalInput, cascadesContext, sort,
+ inputChildrenScope, inputScope);
Builder<OrderKey> boundOrderKeys = ImmutableList.builderWithExpectedSize(sort.getOrderKeys().size());
+ FunctionRegistry functionRegistry = cascadesContext.getConnectContext().getEnv().getFunctionRegistry();
for (OrderKey orderKey : sort.getOrderKeys()) {
- Expression boundKey = bindWithOrdinal(orderKey.getExpr(), analyzer, childOutput);
+ Expression boundKey;
+ if (hasAggregateFunction(orderKey.getExpr(), functionRegistry)) {
+ boundKey = bindInInputChildScope.analyze(orderKey.getExpr());
+ } else {
+ boundKey = bindWithOrdinal(orderKey.getExpr(), bindInInputScopeThenInputChildScope, childOutput);
+ }
boundOrderKeys.add(orderKey.withExpression(boundKey));
}
return new LogicalSort<>(boundOrderKeys.build(), sort.child());
@@ -965,4 +973,49 @@ public class BindExpression implements AnalysisRuleFactory {
private interface CustomSlotBinderAnalyzer {
List<? extends Expression> bindSlot(ExpressionAnalyzer analyzer, UnboundSlot unboundSlot);
}
+
+ private boolean hasAggregateFunction(Expression expression, FunctionRegistry functionRegistry) {
+ return expression.anyMatch(expr -> {
+ if (expr instanceof AggregateFunction) {
+ return true;
+ } else if (expr instanceof UnboundFunction) {
+ UnboundFunction unboundFunction = (UnboundFunction) expr;
+ boolean isAggregateFunction = functionRegistry
+ .isAggregateFunction(
+ unboundFunction.getDbName(),
+ unboundFunction.getName()
+ );
+ return isAggregateFunction;
+ }
+ return false;
+ });
+ }
+
+ private SimpleExprAnalyzer getAnalyzerForOrderByAggFunc(Plan finalInput, CascadesContext cascadesContext,
+ LogicalSort<Plan> sort, Supplier<Scope> inputChildrenScope, Scope inputScope) {
+ ImmutableList.Builder<Slot> outputSlots = ImmutableList.builder();
+ if (finalInput instanceof LogicalAggregate) {
+ LogicalAggregate<Plan> aggregate = (LogicalAggregate<Plan>) finalInput;
+ List<NamedExpression> outputExpressions = aggregate.getOutputExpressions();
+ for (NamedExpression outputExpr : outputExpressions) {
+ if (!outputExpr.anyMatch(expr -> expr instanceof AggregateFunction)) {
+ outputSlots.add(outputExpr.toSlot());
+ }
+ }
+ }
+ Scope outputWithoutAggFunc = toScope(cascadesContext, outputSlots.build());
+ SimpleExprAnalyzer bindInInputChildScope = buildCustomSlotBinderAnalyzer(
+ sort, cascadesContext, inputScope, true, false,
+ (analyzer, unboundSlot) -> {
+ if (finalInput instanceof LogicalAggregate) {
+ List<Slot> boundInOutputWithoutAggFunc = analyzer.bindSlotByScope(unboundSlot,
+ outputWithoutAggFunc);
+ if (!boundInOutputWithoutAggFunc.isEmpty()) {
+ return ImmutableList.of(boundInOutputWithoutAggFunc.get(0));
+ }
+ }
+ return analyzer.bindExactSlotsByThisScope(unboundSlot, inputChildrenScope.get());
+ });
+ return bindInInputChildScope;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
index 82468978a80..82de4453c13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
@@ -218,7 +218,7 @@ public class FillUpMissingSlots implements AnalysisRuleFactory {
// We couldn't find the equivalent expression in output expressions and group-by expressions,
// so we should check whether the expression is valid.
if (expression instanceof SlotReference) {
- throw new AnalysisException(expression.toSql() + " in having clause should be grouped by.");
+ throw new AnalysisException(expression.toSql() + " should be grouped by.");
} else if (expression instanceof AggregateFunction) {
if (checkWhetherNestedAggregateFunctionsExist((AggregateFunction) expression)) {
throw new AnalysisException("Aggregate functions in having clause can't be nested: "
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java
index dbf65decef9..58ce64c09b2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java
@@ -306,7 +306,7 @@ public class FillUpMissingSlotsTest extends AnalyzeCheckTestBase implements Memo
void testInvalidHaving() {
ExceptionChecker.expectThrowsWithMsg(
AnalysisException.class,
- "a2 in having clause should be grouped by.",
+ "a2 should be grouped by.",
() -> PlanChecker.from(connectContext).analyze(
"SELECT a1 FROM t1 GROUP BY a1 HAVING a2 > 0"
));
diff --git a/regression-test/data/nereids_syntax_p0/order_by_bind_priority.out b/regression-test/data/nereids_syntax_p0/order_by_bind_priority.out
new file mode 100644
index 00000000000..3c7dcb1d318
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/order_by_bind_priority.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_bind_order_by_with_aggfun1 --
+10 -5 -10
+4 -2 -4
+4 1 3
+6 3 6
+
+-- !test_bind_order_by_with_aggfun2 --
+4 -2 -4
+10 -5 0
+4 1 4
+6 3 6
+
+-- !test_bind_order_by_with_aggfun3 --
+5 -5 5
+2 -2 -2
+2 1 4
+3 3 3
+
+-- !test_bind_order_by_in_no_agg_func_output --
+1 4
+2 -2
+3 3
+5 5
+
diff --git a/regression-test/suites/nereids_syntax_p0/order_by_bind_priority.groovy b/regression-test/suites/nereids_syntax_p0/order_by_bind_priority.groovy
new file mode 100644
index 00000000000..e434ab42092
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/order_by_bind_priority.groovy
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("order_by_bind_priority") {
+ sql "SET enable_nereids_planner=true;"
+ sql "SET enable_fallback_to_original_planner=false;"
+
+ sql "drop table if exists t_order_by_bind_priority"
+ sql """create table t_order_by_bind_priority (c1 int, c2 int) distributed by hash(c1) properties("replication_num"="1");"""
+ sql "insert into t_order_by_bind_priority values(-2, -2),(1,2),(1,2),(3,3),(-5,5);"
+ sql "sync"
+
+
+ qt_test_bind_order_by_with_aggfun1 "select 2*abs(sum(c1)) as c1, c1,sum(c1)+c1 from t_order_by_bind_priority group by c1 order by sum(c1)+c1 asc;"
+ qt_test_bind_order_by_with_aggfun2 "select 2*abs(sum(c1)) as c2, c1,sum(c1)+c2 from t_order_by_bind_priority group by c1,c2 order by sum(c1)+c2 asc;"
+ qt_test_bind_order_by_with_aggfun3 "select abs(sum(c1)) as c1, c1,sum(c2) as c2 from t_order_by_bind_priority group by c1 order by sum(c1) asc;"
+ qt_test_bind_order_by_in_no_agg_func_output "select abs(c1) xx, sum(c2) from t_order_by_bind_priority group by xx order by min(xx)"
+ test {
+ sql "select abs(sum(c1)) as c1, c1,sum(c2) as c2 from t_order_by_bind_priority group by c1 order by sum(c1)+c2 asc;"
+ exception "c2 should be grouped by."
+ }
+
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/13: [Fix](nereids) forward the user define variables to master (#33013)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 67bb5196137820a766568c520e9faa926166b2c1
Author: feiniaofeiafei <53...@users.noreply.github.com>
AuthorDate: Wed Apr 3 15:31:31 2024 +0800
[Fix](nereids) forward the user define variables to master (#33013)
---
.../org/apache/doris/analysis/LiteralExpr.java | 20 +++++++++++++++++
.../java/org/apache/doris/qe/ConnectContext.java | 8 +++++++
.../java/org/apache/doris/qe/ConnectProcessor.java | 25 +++++++++++++++++++++-
.../java/org/apache/doris/qe/MasterOpExecutor.java | 17 ++++++++++++++-
gensrc/thrift/FrontendService.thrift | 3 ++-
5 files changed, 70 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
index 0814235f0a3..eb6fedb7b50 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
@@ -26,6 +26,8 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.mysql.MysqlProto;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
@@ -478,4 +480,22 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
}
return isZero;
}
+
+ public static LiteralExpr getLiteralExprFromThrift(TExprNode node) throws AnalysisException {
+ TExprNodeType type = node.node_type;
+ switch (type) {
+ case NULL_LITERAL: return new NullLiteral();
+ case BOOL_LITERAL: return new BoolLiteral(node.bool_literal.value);
+ case INT_LITERAL: return new IntLiteral(node.int_literal.value);
+ case LARGE_INT_LITERAL: return new LargeIntLiteral(node.large_int_literal.value);
+ case FLOAT_LITERAL: return new FloatLiteral(node.float_literal.value);
+ case DECIMAL_LITERAL: return new DecimalLiteral(node.decimal_literal.value);
+ case STRING_LITERAL: return new StringLiteral(node.string_literal.value);
+ case JSON_LITERAL: return new JsonLiteral(node.json_literal.value);
+ case DATE_LITERAL: return new DateLiteral(node.date_literal.value);
+ case IPV4_LITERAL: return new IPv4Literal(node.ipv4_literal.value);
+ case IPV6_LITERAL: return new IPv6Literal(node.ipv6_literal.value);
+ default: throw new AnalysisException("Wrong type from thrift;");
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 17a3db0b7fc..a36d4aeacc9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -1128,4 +1128,12 @@ public class ConnectContext {
public void setGroupCommitStreamLoadSql(boolean groupCommitStreamLoadSql) {
isGroupCommitStreamLoadSql = groupCommitStreamLoadSql;
}
+
+ public Map<String, LiteralExpr> getUserVars() {
+ return userVars;
+ }
+
+ public void setUserVars(Map<String, LiteralExpr> userVars) {
+ this.userVars = userVars;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 6bf5aec217a..923dd2b6af1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -19,6 +19,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
@@ -56,22 +57,26 @@ import org.apache.doris.plugin.DialectConverterPlugin;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.proto.Data;
import org.apache.doris.qe.QueryState.MysqlStateType;
+import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TMasterOpRequest;
import org.apache.doris.thrift.TMasterOpResult;
import org.apache.doris.thrift.TUniqueId;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.thrift.TException;
import java.io.IOException;
import java.io.StringReader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import javax.annotation.Nullable;
@@ -509,7 +514,7 @@ public abstract class ConnectProcessor {
}
}
- public TMasterOpResult proxyExecute(TMasterOpRequest request) {
+ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException {
ctx.setDatabase(request.db);
ctx.setQualifiedUser(request.user);
ctx.setEnv(Env.getCurrentEnv());
@@ -564,6 +569,10 @@ public abstract class ConnectProcessor {
}
}
+ if (request.isSetUserVariables()) {
+ ctx.setUserVars(userVariableFromThrift(request.getUserVariables()));
+ }
+
ctx.setThreadLocalInfo();
StmtExecutor executor = null;
try {
@@ -638,4 +647,18 @@ public abstract class ConnectProcessor {
public void processOnce() throws IOException, NotImplementedException {
throw new NotImplementedException("Not Impl processOnce");
}
+
+ private Map<String, LiteralExpr> userVariableFromThrift(Map<String, TExprNode> thriftMap) throws TException {
+ try {
+ Map<String, LiteralExpr> userVariables = Maps.newHashMap();
+ for (Map.Entry<String, TExprNode> entry : thriftMap.entrySet()) {
+ TExprNode tExprNode = entry.getValue();
+ LiteralExpr literalExpr = LiteralExpr.getLiteralExprFromThrift(tExprNode);
+ userVariables.put(entry.getKey(), literalExpr);
+ }
+ return userVariables;
+ } catch (AnalysisException e) {
+ throw new TException(e.getMessage());
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
index 40c126b732d..934c221905f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
@@ -17,18 +17,22 @@
package org.apache.doris.qe;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.RedirectStatus;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.thrift.FrontendService;
+import org.apache.doris.thrift.TExpr;
+import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TMasterOpRequest;
import org.apache.doris.thrift.TMasterOpResult;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TUniqueId;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TException;
@@ -163,7 +167,7 @@ public class MasterOpExecutor {
params.setQueryOptions(ctx.getSessionVariable().getQueryOptionVariables());
// session variables
params.setSessionVariables(ctx.getSessionVariable().getForwardVariables());
-
+ params.setUserVariables(getForwardUserVariables(ctx.getUserVars()));
if (null != ctx.queryId()) {
params.setQueryId(ctx.queryId());
}
@@ -268,4 +272,15 @@ public class MasterOpExecutor {
return msg;
}
}
+
+ private Map<String, TExprNode> getForwardUserVariables(Map<String, LiteralExpr> userVariables) {
+ Map<String, TExprNode> forwardVariables = Maps.newHashMap();
+ for (Map.Entry<String, LiteralExpr> entry : userVariables.entrySet()) {
+ LiteralExpr literalExpr = entry.getValue();
+ TExpr tExpr = literalExpr.treeToThrift();
+ TExprNode tExprNode = tExpr.nodes.get(0);
+ forwardVariables.put(entry.getKey(), tExprNode);
+ }
+ return forwardVariables;
+ }
}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 049c8450b23..320149855b4 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -490,7 +490,6 @@ struct TFeResult {
1: required FrontendServiceVersion protocolVersion
2: required Status.TStatus status
}
-
struct TMasterOpRequest {
1: required string user
2: required string db
@@ -520,6 +519,8 @@ struct TMasterOpRequest {
24: optional bool syncJournalOnly // if set to true, this request means to do nothing but just sync max journal id of master
25: optional string defaultCatalog
26: optional string defaultDatabase
+ 27: optional bool cancel_qeury // if set to true, this request means to cancel one forwarded query, and query_id needs to be set
+ 28: optional map<string, Exprs.TExprNode> user_variables
}
struct TColumnDefinition {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/13: [enhance](mtmv)support create mtmv with other mtmv (#32984)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0ab8b57db7c0c7c01fc72a6f12b1a7f3f8b344bc
Author: zhangdong <49...@qq.com>
AuthorDate: Wed Apr 3 11:49:24 2024 +0800
[enhance](mtmv)support create mtmv with other mtmv (#32984)
---
.../trees/plans/commands/info/CreateMTMVInfo.java | 11 ---
.../data/mtmv_p0/test_multi_level_mtmv.out | 13 ++++
.../suites/mtmv_p0/test_build_mtmv.groovy | 15 ----
.../suites/mtmv_p0/test_multi_level_mtmv.groovy | 81 ++++++++++++++++++++++
4 files changed, 94 insertions(+), 26 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 7e7f58db44d..3846cd99ff1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -29,7 +29,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.View;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
@@ -64,8 +63,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
-import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
-import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
@@ -319,14 +316,6 @@ public class CreateMTMVInfo {
}
private void analyzeBaseTables(Plan plan) {
- TableCollectorContext collectorContext =
- new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.MATERIALIZED_VIEW), true);
- plan.accept(TableCollector.INSTANCE, collectorContext);
- Set<TableIf> collectedTables = collectorContext.getCollectedTables();
- if (!CollectionUtils.isEmpty(collectedTables)) {
- throw new AnalysisException("can not contain MATERIALIZED_VIEW");
- }
-
List<Object> subQuerys = plan.collectToList(node -> node instanceof LogicalSubQueryAlias);
for (Object subquery : subQuerys) {
List<String> qualifier = ((LogicalSubQueryAlias) subquery).getQualifier();
diff --git a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
new file mode 100644
index 00000000000..3d8a1e1b43f
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !mv1 --
+1 1
+
+-- !mv2 --
+1 1
+
+-- !status1 --
+multi_level_mtmv1 SCHEMA_CHANGE SUCCESS
+
+-- !status2 --
+multi_level_mtmv2 SCHEMA_CHANGE SUCCESS
+
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index 141a324e979..ed5f11aeb1a 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -122,21 +122,6 @@ suite("test_build_mtmv") {
log.info(e.getMessage())
}
- // not allow create mv use other mv
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvNameRenamed}
- BUILD DEFERRED REFRESH COMPLETE ON MANUAL
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES ('replication_num' = '1')
- AS
- SELECT * from ${mvName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- }
-
// not allow create mv use view
try {
sql """
diff --git a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
new file mode 100644
index 00000000000..81d4b78a3d7
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_multi_level_mtmv") {
+ def tableName = "t_test_multi_level_user"
+ def mv1 = "multi_level_mtmv1"
+ def mv2 = "multi_level_mtmv2"
+ def dbName = "regression_test_mtmv_p0"
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mv1};"""
+ sql """drop materialized view if exists ${mv2};"""
+
+ sql """
+ CREATE TABLE IF NOT EXISTS `${tableName}` (
+ k1 int,
+ k2 int
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ INSERT INTO ${tableName} VALUES(1,1);
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mv1}
+ BUILD DEFERRED REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT * FROM ${tableName};
+ """
+ def jobName1 = getJobName("regression_test_mtmv_p0", mv1);
+ sql """
+ REFRESH MATERIALIZED VIEW ${mv1} AUTO
+ """
+ waitingMTMVTaskFinished(jobName1)
+ order_qt_mv1 "select * from ${mv1}"
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mv2}
+ BUILD DEFERRED REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT * FROM ${mv1};
+ """
+ def jobName2 = getJobName("regression_test_mtmv_p0", mv2);
+ sql """
+ REFRESH MATERIALIZED VIEW ${mv2} AUTO
+ """
+ waitingMTMVTaskFinished(jobName2)
+ order_qt_mv2 "select * from ${mv2}"
+
+ // drop table
+ sql """
+ drop table ${tableName}
+ """
+ order_qt_status1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mv1}'"
+ order_qt_status2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mv2}'"
+
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mv1};"""
+ sql """drop materialized view if exists ${mv2};"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/13: [FIX](inverted_index) fix inverted index write array with _doc is empty (#33170)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7fae123b01f74777fa6446514c98ebb47b164dab
Author: amory <wa...@selectdb.com>
AuthorDate: Wed Apr 3 20:04:59 2024 +0800
[FIX](inverted_index) fix inverted index write array with _doc is empty (#33170)
---
be/src/clucene | 2 +-
be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/be/src/clucene b/be/src/clucene
index ff2cd82f9e5..9f849a47f70 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit ff2cd82f9e545a24318f1256eba312b4d0562a82
+Subproject commit 9f849a47f70625a57fedbaa1f5a6f89bc8f32967
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index fe1c0c4f9ac..641eaf2b886 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -401,9 +401,12 @@ public:
_doc->add(*new_field);
}
}
- RETURN_IF_ERROR(add_document());
- _doc->clear();
- _CLDELETE(ts);
+ if (!_doc->getFields()->empty()) {
+ // if this array is null, we just ignore to write inverted index
+ RETURN_IF_ERROR(add_document());
+ _doc->clear();
+ _CLDELETE(ts);
+ }
_rid++;
}
} else if constexpr (field_is_numeric_type(field_type)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/13: [fix](Nereids) convert agg state type failed in some cases (#33208)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit edd170196334672a0263dd5177ad82305753676b
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Wed Apr 3 16:24:36 2024 +0800
[fix](Nereids) convert agg state type failed in some cases (#33208)
---
.../expression/rules/ConvertAggStateCast.java | 38 +++++++++++++---------
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java
index 23900701553..6aa4529ddd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.combinator.StateCombinator;
@@ -42,7 +43,7 @@ public class ConvertAggStateCast implements ExpressionPatternRuleFactory {
@Override
public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
return ImmutableList.of(
- matchesTopType(Cast.class).then(ConvertAggStateCast::convert)
+ matchesType(Cast.class).then(ConvertAggStateCast::convert)
);
}
@@ -50,24 +51,29 @@ public class ConvertAggStateCast implements ExpressionPatternRuleFactory {
Expression child = cast.child();
DataType originalType = child.getDataType();
DataType targetType = cast.getDataType();
- if (originalType instanceof AggStateType
- && targetType instanceof AggStateType
- && child instanceof StateCombinator) {
- AggStateType target = (AggStateType) targetType;
- ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(child.arity());
- for (int i = 0; i < child.arity(); i++) {
- Expression newChild = TypeCoercionUtils.castIfNotSameType(child.child(i), target.getSubTypes().get(i));
- if (newChild.nullable() != target.getSubTypeNullables().get(i)) {
- if (newChild.nullable()) {
- newChild = new NonNullable(newChild);
- } else {
- newChild = new Nullable(newChild);
+ if (originalType instanceof AggStateType && targetType instanceof AggStateType) {
+ // TODO remve it after we refactor mv rewriter to avoid generate Alias in expression
+ while (child instanceof Alias) {
+ child = ((Alias) child).child();
+ }
+ if (child instanceof StateCombinator) {
+ AggStateType target = (AggStateType) targetType;
+ ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(child.arity());
+ for (int i = 0; i < child.arity(); i++) {
+ Expression newChild = TypeCoercionUtils.castIfNotSameTypeStrict(
+ child.child(i), target.getSubTypes().get(i));
+ if (newChild.nullable() != target.getSubTypeNullables().get(i)) {
+ if (newChild.nullable()) {
+ newChild = new NonNullable(newChild);
+ } else {
+ newChild = new Nullable(newChild);
+ }
}
+ newChildren.add(newChild);
}
- newChildren.add(newChild);
+ child = child.withChildren(newChildren.build());
+ return cast.withChildren(ImmutableList.of(child));
}
- child = child.withChildren(newChildren.build());
- return cast.withChildren(ImmutableList.of(child));
}
return cast;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/13: [fix](Nereids) support update without filter (#33214)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 96867ff3fd7e94d9142df72264320d7442ed084e
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Wed Apr 3 19:04:10 2024 +0800
[fix](Nereids) support update without filter (#33214)
---
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 0fbb0a1f960..9f911d71fe1 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -69,7 +69,7 @@ statementBase
| explain? cte? UPDATE tableName=multipartIdentifier tableAlias
SET updateAssignmentSeq
fromClause?
- whereClause #update
+ whereClause? #update
| explain? cte? DELETE FROM tableName=multipartIdentifier
partitionSpec? tableAlias
(USING relations)?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 5492e7de5be..247df0e2d68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -828,7 +828,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
if (ctx.fromClause() != null) {
query = withRelations(query, ctx.fromClause().relations().relation());
}
- query = withFilter(query, Optional.of(ctx.whereClause()));
+ query = withFilter(query, Optional.ofNullable(ctx.whereClause()));
String tableAlias = null;
if (ctx.tableAlias().strictIdentifier() != null) {
tableAlias = ctx.tableAlias().getText();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/13: [chore](spill) add timers for performance tuning (#33185)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2e40e395844cf716a91f09c4be9bcf7b583aefbd
Author: Jerry Hu <mr...@gmail.com>
AuthorDate: Wed Apr 3 15:19:04 2024 +0800
[chore](spill) add timers for performance tuning (#33185)
---
.../exec/partitioned_aggregation_sink_operator.cpp | 5 +-
.../partitioned_aggregation_source_operator.cpp | 7 +-
.../exec/partitioned_hash_join_probe_operator.cpp | 100 ++++++++++++---------
.../exec/partitioned_hash_join_probe_operator.h | 24 +++--
.../exec/partitioned_hash_join_sink_operator.cpp | 53 +++++------
.../exec/partitioned_hash_join_sink_operator.h | 12 +--
be/src/pipeline/exec/spill_sort_sink_operator.cpp | 10 ++-
.../pipeline/exec/spill_sort_source_operator.cpp | 9 +-
be/src/pipeline/pipeline_x/operator.h | 6 ++
9 files changed, 137 insertions(+), 89 deletions(-)
diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
index 64996724e15..3dea330c117 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp
@@ -252,13 +252,16 @@ Status PartitionedAggSinkLocalState::revoke_memory(RuntimeState* state) {
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
status = ExecEnv::GetInstance()->spill_stream_mgr()->get_async_task_thread_pool()->submit_func(
- [this, &parent, state, execution_context] {
+ [this, &parent, state, execution_context, submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return Status::Cancelled("Cancelled");
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
SCOPED_ATTACH_TASK(state);
SCOPED_TIMER(Base::_spill_timer);
Defer defer {[&]() {
diff --git a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
index 5db80788f41..5680b75c87e 100644
--- a/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_aggregation_source_operator.cpp
@@ -197,9 +197,13 @@ Status PartitionedAggLocalState::initiate_merge_spill_partition_agg_data(Runtime
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
RETURN_IF_ERROR(
ExecEnv::GetInstance()->spill_stream_mgr()->get_async_task_thread_pool()->submit_func(
- [this, state, execution_context] {
+ [this, state, execution_context, submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
@@ -207,6 +211,7 @@ Status PartitionedAggLocalState::initiate_merge_spill_partition_agg_data(Runtime
return Status::Cancelled("Cancelled");
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
SCOPED_ATTACH_TASK(state);
Defer defer {[&]() {
if (!_status.ok()) {
diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
index c23e12c3705..1a05b78b052 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp
@@ -25,10 +25,11 @@ namespace doris::pipeline {
PartitionedHashJoinProbeLocalState::PartitionedHashJoinProbeLocalState(RuntimeState* state,
OperatorXBase* parent)
- : JoinProbeLocalState(state, parent) {}
+ : PipelineXSpillLocalState(state, parent),
+ _child_block(vectorized::Block::create_unique()) {}
Status PartitionedHashJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo& info) {
- RETURN_IF_ERROR(JoinProbeLocalState::init(state, info));
+ RETURN_IF_ERROR(PipelineXSpillLocalState::init(state, info));
_internal_runtime_profile.reset(new RuntimeProfile("internal_profile"));
auto& p = _parent->cast<PartitionedHashJoinProbeOperatorX>();
@@ -38,45 +39,32 @@ Status PartitionedHashJoinProbeLocalState::init(RuntimeState* state, LocalStateI
RETURN_IF_ERROR(_partitioner->init(p._probe_exprs));
RETURN_IF_ERROR(_partitioner->prepare(state, p._child_x->row_desc()));
- _spill_and_partition_label = ADD_LABEL_COUNTER(profile(), "SpillAndPartition");
- _partition_timer = ADD_CHILD_TIMER(profile(), "PartitionTime", "SpillAndPartition");
- _partition_shuffle_timer =
- ADD_CHILD_TIMER(profile(), "PartitionShuffleTime", "SpillAndPartition");
- _spill_build_rows =
- ADD_CHILD_COUNTER(profile(), "SpillBuildRows", TUnit::UNIT, "SpillAndPartition");
- _recovery_build_rows =
- ADD_CHILD_COUNTER(profile(), "RecoveryBuildRows", TUnit::UNIT, "SpillAndPartition");
- _spill_probe_rows =
- ADD_CHILD_COUNTER(profile(), "SpillProbeRows", TUnit::UNIT, "SpillAndPartition");
- _recovery_probe_rows =
- ADD_CHILD_COUNTER(profile(), "RecoveryProbeRows", TUnit::UNIT, "SpillAndPartition");
- _spill_build_blocks =
- ADD_CHILD_COUNTER(profile(), "SpillBuildBlocks", TUnit::UNIT, "SpillAndPartition");
+ _spill_and_partition_label = ADD_LABEL_COUNTER(profile(), "Partition");
+ _partition_timer = ADD_CHILD_TIMER(profile(), "PartitionTime", "Partition");
+ _partition_shuffle_timer = ADD_CHILD_TIMER(profile(), "PartitionShuffleTime", "Partition");
+ _spill_build_rows = ADD_CHILD_COUNTER(profile(), "SpillBuildRows", TUnit::UNIT, "Spill");
+ _spill_build_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "SpillBuildTime", "Spill", 1);
+ _recovery_build_rows = ADD_CHILD_COUNTER(profile(), "RecoveryBuildRows", TUnit::UNIT, "Spill");
+ _recovery_build_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "RecoveryBuildTime", "Spill", 1);
+ _spill_probe_rows = ADD_CHILD_COUNTER(profile(), "SpillProbeRows", TUnit::UNIT, "Spill");
+ _recovery_probe_rows = ADD_CHILD_COUNTER(profile(), "RecoveryProbeRows", TUnit::UNIT, "Spill");
+ _spill_build_blocks = ADD_CHILD_COUNTER(profile(), "SpillBuildBlocks", TUnit::UNIT, "Spill");
_recovery_build_blocks =
- ADD_CHILD_COUNTER(profile(), "RecoveryBuildBlocks", TUnit::UNIT, "SpillAndPartition");
- _spill_probe_blocks =
- ADD_CHILD_COUNTER(profile(), "SpillProbeBlocks", TUnit::UNIT, "SpillAndPartition");
+ ADD_CHILD_COUNTER(profile(), "RecoveryBuildBlocks", TUnit::UNIT, "Spill");
+ _spill_probe_blocks = ADD_CHILD_COUNTER(profile(), "SpillProbeBlocks", TUnit::UNIT, "Spill");
+ _spill_probe_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "SpillProbeTime", "Spill", 1);
_recovery_probe_blocks =
- ADD_CHILD_COUNTER(profile(), "RecoveryProbeBlocks", TUnit::UNIT, "SpillAndPartition");
+ ADD_CHILD_COUNTER(profile(), "RecoveryProbeBlocks", TUnit::UNIT, "Spill");
+ _recovery_probe_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "RecoveryProbeTime", "Spill", 1);
- _spill_serialize_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(
- Base::profile(), "SpillSerializeBlockTime", "SpillAndPartition", 1);
- _spill_write_disk_timer = ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWriteDiskTime",
- "SpillAndPartition", 1);
+ _spill_serialize_block_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillSerializeBlockTime", "Spill", 1);
+ _spill_write_disk_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWriteDiskTime", "Spill", 1);
_spill_data_size = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillWriteDataSize",
- TUnit::BYTES, "SpillAndPartition", 1);
+ TUnit::BYTES, "Spill", 1);
_spill_block_count = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillWriteBlockCount",
- TUnit::UNIT, "SpillAndPartition", 1);
- _spill_read_data_time = ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillReadDataTime",
- "SpillAndPartition", 1);
- _spill_deserialize_time = ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillDeserializeTime",
- "SpillAndPartition", 1);
- _spill_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillReadDataSize",
- TUnit::BYTES, "SpillAndPartition", 1);
- _spill_write_wait_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWriteWaitIOTime",
- "SpillAndPartition", 1);
- _spill_read_wait_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillReadWaitIOTime",
- "SpillAndPartition", 1);
+ TUnit::UNIT, "Spill", 1);
// Build phase
_build_phase_label = ADD_LABEL_COUNTER(profile(), "BuildPhase");
@@ -109,6 +97,10 @@ Status PartitionedHashJoinProbeLocalState::init(RuntimeState* state, LocalStateI
_process_other_join_conjunct_timer =
ADD_CHILD_TIMER(profile(), "OtherJoinConjunctTime", "ProbePhase");
_init_probe_side_timer = ADD_CHILD_TIMER(profile(), "InitProbeSideTime", "ProbePhase");
+ _probe_timer = ADD_CHILD_TIMER(profile(), "ProbeTime", "ProbePhase");
+ _join_filter_timer = ADD_CHILD_TIMER(profile(), "JoinFilterTimer", "ProbePhase");
+ _build_output_block_timer = ADD_CHILD_TIMER(profile(), "BuildOutputBlock", "ProbePhase");
+ _probe_rows_counter = ADD_CHILD_COUNTER(profile(), "ProbeRows", TUnit::UNIT, "ProbePhase");
return Status::OK();
}
#define UPDATE_PROFILE(counter, name) \
@@ -149,7 +141,7 @@ void PartitionedHashJoinProbeLocalState::update_probe_profile(RuntimeProfile* ch
#undef UPDATE_PROFILE
Status PartitionedHashJoinProbeLocalState::open(RuntimeState* state) {
- RETURN_IF_ERROR(PipelineXLocalStateBase::open(state));
+ RETURN_IF_ERROR(PipelineXSpillLocalState::open(state));
return _partitioner->open(state);
}
Status PartitionedHashJoinProbeLocalState::close(RuntimeState* state) {
@@ -157,7 +149,7 @@ Status PartitionedHashJoinProbeLocalState::close(RuntimeState* state) {
return Status::OK();
}
dec_running_big_mem_op_num(state);
- RETURN_IF_ERROR(JoinProbeLocalState::close(state));
+ RETURN_IF_ERROR(PipelineXSpillLocalState::close(state));
return Status::OK();
}
@@ -187,13 +179,17 @@ Status PartitionedHashJoinProbeLocalState::spill_build_block(RuntimeState* state
build_spilling_stream->get_spill_root_dir());
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
return spill_io_pool->submit_func(
- [execution_context, state, &build_spilling_stream, &mutable_block, this] {
+ [execution_context, state, &build_spilling_stream, &mutable_block, submit_timer, this] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return;
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
+ SCOPED_TIMER(_spill_build_timer);
(void)state; // avoid ut compile error
SCOPED_ATTACH_TASK(state);
if (_spill_status_ok) {
@@ -248,14 +244,18 @@ Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* stat
if (!blocks.empty()) {
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
return spill_io_pool->submit_func(
- [execution_context, state, &blocks, spilling_stream, this] {
+ [execution_context, state, &blocks, spilling_stream, submit_timer, this] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
_dependency->set_ready();
return;
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
+ SCOPED_TIMER(_spill_probe_timer);
SCOPED_ATTACH_TASK(state);
COUNTER_UPDATE(_spill_probe_blocks, blocks.size());
while (!blocks.empty() && !state->is_cancelled()) {
@@ -329,12 +329,19 @@ Status PartitionedHashJoinProbeLocalState::recovery_build_blocks_from_disk(Runti
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
- auto read_func = [this, state, &spilled_stream, &mutable_block, execution_context] {
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
+ auto read_func = [this, state, &spilled_stream, &mutable_block, execution_context,
+ submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return;
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
+ SCOPED_TIMER(_recovery_build_timer);
Defer defer([this] { --_spilling_task_count; });
(void)state; // avoid ut compile error
SCOPED_ATTACH_TASK(state);
@@ -403,12 +410,19 @@ Status PartitionedHashJoinProbeLocalState::recovery_probe_blocks_from_disk(Runti
/// TODO: maybe recovery more blocks each time.
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
- auto read_func = [this, execution_context, state, &spilled_stream, &blocks] {
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
+ auto read_func = [this, execution_context, state, &spilled_stream, &blocks, submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return;
}
+
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
+ SCOPED_TIMER(_recovery_probe_timer);
Defer defer([this] { --_spilling_task_count; });
(void)state; // avoid ut compile error
SCOPED_ATTACH_TASK(state);
@@ -827,4 +841,4 @@ Status PartitionedHashJoinProbeOperatorX::get_block(RuntimeState* state, vectori
return Status::OK();
}
-} // namespace doris::pipeline
\ No newline at end of file
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
index 8270817758d..143576e1b86 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
+++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h
@@ -38,16 +38,13 @@ using PartitionerType = vectorized::XXHashPartitioner<LocalExchangeChannelIds>;
class PartitionedHashJoinProbeOperatorX;
class PartitionedHashJoinProbeLocalState final
- : public JoinProbeLocalState<PartitionedHashJoinSharedState,
- PartitionedHashJoinProbeLocalState> {
+ : public PipelineXSpillLocalState<PartitionedHashJoinSharedState> {
public:
using Parent = PartitionedHashJoinProbeOperatorX;
ENABLE_FACTORY_CREATOR(PartitionedHashJoinProbeLocalState);
PartitionedHashJoinProbeLocalState(RuntimeState* state, OperatorXBase* parent);
~PartitionedHashJoinProbeLocalState() override = default;
- void add_tuple_is_null_column(vectorized::Block* block) override {}
-
Status init(RuntimeState* state, LocalStateInfo& info) override;
Status open(RuntimeState* state) override;
Status close(RuntimeState* state) override;
@@ -68,9 +65,15 @@ public:
friend class PartitionedHashJoinProbeOperatorX;
private:
+ template <typename LocalStateType>
+ friend class StatefulOperatorX;
+
std::shared_ptr<BasicSharedState> _in_mem_shared_state_sptr;
uint32_t _partition_cursor {0};
+ std::unique_ptr<vectorized::Block> _child_block;
+ bool _child_eos {false};
+
std::mutex _spill_lock;
Status _spill_status;
@@ -98,22 +101,21 @@ private:
RuntimeProfile::Counter* _partition_shuffle_timer = nullptr;
RuntimeProfile::Counter* _spill_build_rows = nullptr;
RuntimeProfile::Counter* _spill_build_blocks = nullptr;
+ RuntimeProfile::Counter* _spill_build_timer = nullptr;
RuntimeProfile::Counter* _recovery_build_rows = nullptr;
RuntimeProfile::Counter* _recovery_build_blocks = nullptr;
+ RuntimeProfile::Counter* _recovery_build_timer = nullptr;
RuntimeProfile::Counter* _spill_probe_rows = nullptr;
RuntimeProfile::Counter* _spill_probe_blocks = nullptr;
+ RuntimeProfile::Counter* _spill_probe_timer = nullptr;
RuntimeProfile::Counter* _recovery_probe_rows = nullptr;
RuntimeProfile::Counter* _recovery_probe_blocks = nullptr;
+ RuntimeProfile::Counter* _recovery_probe_timer = nullptr;
- RuntimeProfile::Counter* _spill_read_data_time = nullptr;
- RuntimeProfile::Counter* _spill_deserialize_time = nullptr;
- RuntimeProfile::Counter* _spill_read_bytes = nullptr;
RuntimeProfile::Counter* _spill_serialize_block_timer = nullptr;
RuntimeProfile::Counter* _spill_write_disk_timer = nullptr;
RuntimeProfile::Counter* _spill_data_size = nullptr;
RuntimeProfile::Counter* _spill_block_count = nullptr;
- RuntimeProfile::Counter* _spill_write_wait_io_timer = nullptr;
- RuntimeProfile::Counter* _spill_read_wait_io_timer = nullptr;
RuntimeProfile::Counter* _build_phase_label = nullptr;
RuntimeProfile::Counter* _build_rows_counter = nullptr;
@@ -137,6 +139,10 @@ private:
RuntimeProfile::Counter* _init_probe_side_timer = nullptr;
RuntimeProfile::Counter* _build_side_output_timer = nullptr;
RuntimeProfile::Counter* _process_other_join_conjunct_timer = nullptr;
+ RuntimeProfile::Counter* _probe_timer = nullptr;
+ RuntimeProfile::Counter* _probe_rows_counter = nullptr;
+ RuntimeProfile::Counter* _join_filter_timer = nullptr;
+ RuntimeProfile::Counter* _build_output_block_timer = nullptr;
};
class PartitionedHashJoinProbeOperatorX final
diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
index f38354d5de2..8b9accd30ad 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
+++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp
@@ -25,7 +25,7 @@ namespace doris::pipeline {
Status PartitionedHashJoinSinkLocalState::init(doris::RuntimeState* state,
doris::pipeline::LocalSinkStateInfo& info) {
- RETURN_IF_ERROR(PipelineXSinkLocalState::init(state, info));
+ RETURN_IF_ERROR(PipelineXSpillSinkLocalState::init(state, info));
auto& p = _parent->cast<PartitionedHashJoinSinkOperatorX>();
_shared_state->partitioned_build_blocks.resize(p._partition_count);
_shared_state->spilled_streams.resize(p._partition_count);
@@ -33,30 +33,26 @@ Status PartitionedHashJoinSinkLocalState::init(doris::RuntimeState* state,
_partitioner = std::make_unique<PartitionerType>(p._partition_count);
RETURN_IF_ERROR(_partitioner->init(p._build_exprs));
- _partition_timer = ADD_TIMER(profile(), "PartitionTime");
- _partition_shuffle_timer = ADD_TIMER(profile(), "PartitionShuffleTime");
-
- _spill_serialize_block_timer = ADD_TIMER_WITH_LEVEL(profile(), "SpillSerializeBlockTime", 1);
- _spill_write_disk_timer = ADD_TIMER_WITH_LEVEL(profile(), "SpillWriteDiskTime", 1);
- _spill_data_size = ADD_COUNTER_WITH_LEVEL(profile(), "SpillWriteDataSize", TUnit::BYTES, 1);
- _spill_block_count = ADD_COUNTER_WITH_LEVEL(profile(), "SpillWriteBlockCount", TUnit::UNIT, 1);
- _spill_write_wait_io_timer = ADD_TIMER_WITH_LEVEL(profile(), "SpillWriteWaitIOTime", 1);
+ _partition_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "PartitionTime", "Spill", 1);
+ _partition_shuffle_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile(), "PartitionShuffleTime", "Spill", 1);
+ _spill_build_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile(), "SpillBuildTime", "Spill", 1);
return _partitioner->prepare(state, p._child_x->row_desc());
}
Status PartitionedHashJoinSinkLocalState::open(RuntimeState* state) {
- RETURN_IF_ERROR(PipelineXSinkLocalState::open(state));
+ RETURN_IF_ERROR(PipelineXSpillSinkLocalState::open(state));
return _partitioner->open(state);
}
Status PartitionedHashJoinSinkLocalState::close(RuntimeState* state, Status exec_status) {
- SCOPED_TIMER(PipelineXSinkLocalState::exec_time_counter());
- SCOPED_TIMER(PipelineXSinkLocalState::_close_timer);
- if (PipelineXSinkLocalState::_closed) {
+ SCOPED_TIMER(PipelineXSpillSinkLocalState::exec_time_counter());
+ SCOPED_TIMER(PipelineXSpillSinkLocalState::_close_timer);
+ if (PipelineXSpillSinkLocalState::_closed) {
return Status::OK();
}
dec_running_big_mem_op_num(state);
- return PipelineXSinkLocalState::close(state, exec_status);
+ return PipelineXSpillSinkLocalState::close(state, exec_status);
}
Status PartitionedHashJoinSinkLocalState::revoke_memory(RuntimeState* state) {
@@ -90,16 +86,23 @@ Status PartitionedHashJoinSinkLocalState::revoke_memory(RuntimeState* state) {
DCHECK(spill_io_pool != nullptr);
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
- auto st = spill_io_pool->submit_func([this, execution_context, state, spilling_stream, i] {
- auto execution_context_lock = execution_context.lock();
- if (!execution_context_lock) {
- LOG(INFO) << "execution_context released, maybe query was cancelled.";
- return;
- }
- (void)state; // avoid ut compile error
- SCOPED_ATTACH_TASK(state);
- _spill_to_disk(i, spilling_stream);
- });
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
+ auto st = spill_io_pool->submit_func(
+ [this, execution_context, state, spilling_stream, i, submit_timer] {
+ auto execution_context_lock = execution_context.lock();
+ if (!execution_context_lock) {
+ LOG(INFO) << "execution_context released, maybe query was cancelled.";
+ return;
+ }
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
+ SCOPED_TIMER(_spill_build_timer);
+ (void)state; // avoid ut compile error
+ SCOPED_ATTACH_TASK(state);
+ _spill_to_disk(i, spilling_stream);
+ });
if (!st.ok()) {
--_spilling_streams_count;
@@ -274,4 +277,4 @@ Status PartitionedHashJoinSinkOperatorX::revoke_memory(RuntimeState* state) {
return local_state.revoke_memory(state);
}
-} // namespace doris::pipeline
\ No newline at end of file
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
index e364e225f66..96e751360d4 100644
--- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
+++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h
@@ -39,7 +39,7 @@ using PartitionerType = vectorized::XXHashPartitioner<LocalExchangeChannelIds>;
class PartitionedHashJoinSinkOperatorX;
class PartitionedHashJoinSinkLocalState
- : public PipelineXSinkLocalState<PartitionedHashJoinSharedState> {
+ : public PipelineXSpillSinkLocalState<PartitionedHashJoinSharedState> {
public:
using Parent = PartitionedHashJoinSinkOperatorX;
ENABLE_FACTORY_CREATOR(PartitionedHashJoinSinkLocalState);
@@ -51,7 +51,7 @@ public:
protected:
PartitionedHashJoinSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
- : PipelineXSinkLocalState<PartitionedHashJoinSharedState>(parent, state) {}
+ : PipelineXSpillSinkLocalState<PartitionedHashJoinSharedState>(parent, state) {}
void _spill_to_disk(uint32_t partition_index,
const vectorized::SpillStreamSPtr& spilling_stream);
@@ -76,11 +76,7 @@ protected:
RuntimeProfile::Counter* _partition_timer = nullptr;
RuntimeProfile::Counter* _partition_shuffle_timer = nullptr;
- RuntimeProfile::Counter* _spill_serialize_block_timer = nullptr;
- RuntimeProfile::Counter* _spill_write_disk_timer = nullptr;
- RuntimeProfile::Counter* _spill_data_size = nullptr;
- RuntimeProfile::Counter* _spill_block_count = nullptr;
- RuntimeProfile::Counter* _spill_write_wait_io_timer = nullptr;
+ RuntimeProfile::Counter* _spill_build_timer = nullptr;
};
class PartitionedHashJoinSinkOperatorX
@@ -139,4 +135,4 @@ private:
};
} // namespace pipeline
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/pipeline/exec/spill_sort_sink_operator.cpp b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
index 662e195f3e5..523ff2cfaaf 100644
--- a/be/src/pipeline/exec/spill_sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/spill_sort_sink_operator.cpp
@@ -55,6 +55,9 @@ void SpillSortSinkLocalState::_init_counters() {
_spill_merge_sort_timer =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SpillMergeSortTime", "Spill", 1);
+
+ _spill_wait_in_queue_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile(), "SpillWaitInQueueTime", "Spill", 1);
}
#define UPDATE_PROFILE(counter, name) \
do { \
@@ -227,17 +230,22 @@ Status SpillSortSinkLocalState::revoke_memory(RuntimeState* state) {
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
status =
ExecEnv::GetInstance()
->spill_stream_mgr()
->get_spill_io_thread_pool(_spilling_stream->get_spill_root_dir())
- ->submit_func([this, state, &parent, execution_context] {
+ ->submit_func([this, state, &parent, execution_context, submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return Status::OK();
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
SCOPED_ATTACH_TASK(state);
Defer defer {[&]() {
if (!_shared_state->sink_status.ok()) {
diff --git a/be/src/pipeline/exec/spill_sort_source_operator.cpp b/be/src/pipeline/exec/spill_sort_source_operator.cpp
index c021687e1df..c53c057088c 100644
--- a/be/src/pipeline/exec/spill_sort_source_operator.cpp
+++ b/be/src/pipeline/exec/spill_sort_source_operator.cpp
@@ -43,6 +43,8 @@ Status SpillSortLocalState::init(RuntimeState* state, LocalStateInfo& info) {
TUnit::BYTES, "Spill", 1);
_spill_block_count = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillWriteBlockCount",
TUnit::UNIT, "Spill", 1);
+ _spill_wait_in_queue_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile(), "SpillWaitInQueueTime", "Spill", 1);
return Status::OK();
}
@@ -82,13 +84,18 @@ Status SpillSortLocalState::initiate_merge_sort_spill_streams(RuntimeState* stat
auto execution_context = state->get_task_execution_context();
_shared_state_holder = _shared_state->shared_from_this();
- auto spill_func = [this, state, &parent, execution_context] {
+
+ MonotonicStopWatch submit_timer;
+ submit_timer.start();
+
+ auto spill_func = [this, state, &parent, execution_context, submit_timer] {
auto execution_context_lock = execution_context.lock();
if (!execution_context_lock) {
LOG(INFO) << "execution_context released, maybe query was cancelled.";
return Status::OK();
}
+ _spill_wait_in_queue_timer->update(submit_timer.elapsed_time());
SCOPED_TIMER(_spill_merge_sort_timer);
SCOPED_ATTACH_TASK(state);
Defer defer {[&]() {
diff --git a/be/src/pipeline/pipeline_x/operator.h b/be/src/pipeline/pipeline_x/operator.h
index 20fa46a5bf9..45e42390bc5 100644
--- a/be/src/pipeline/pipeline_x/operator.h
+++ b/be/src/pipeline/pipeline_x/operator.h
@@ -455,6 +455,8 @@ public:
ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillDeserializeTime", "Spill", 1);
_spill_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillReadDataSize",
TUnit::BYTES, "Spill", 1);
+ _spill_wait_in_queue_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWaitInQueueTime", "Spill", 1);
_spill_write_wait_io_timer =
ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWriteWaitIOTime", "Spill", 1);
_spill_read_wait_io_timer =
@@ -469,6 +471,7 @@ public:
RuntimeProfile::Counter* _spill_read_bytes;
RuntimeProfile::Counter* _spill_write_wait_io_timer = nullptr;
RuntimeProfile::Counter* _spill_read_wait_io_timer = nullptr;
+ RuntimeProfile::Counter* _spill_wait_in_queue_timer = nullptr;
};
class DataSinkOperatorXBase;
@@ -776,6 +779,8 @@ public:
TUnit::BYTES, "Spill", 1);
_spill_block_count = ADD_CHILD_COUNTER_WITH_LEVEL(Base::profile(), "SpillWriteBlockCount",
TUnit::UNIT, "Spill", 1);
+ _spill_wait_in_queue_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWaitInQueueTime", "Spill", 1);
_spill_write_wait_io_timer =
ADD_CHILD_TIMER_WITH_LEVEL(Base::profile(), "SpillWriteWaitIOTime", "Spill", 1);
_spill_read_wait_io_timer =
@@ -789,6 +794,7 @@ public:
RuntimeProfile::Counter* _spill_write_disk_timer = nullptr;
RuntimeProfile::Counter* _spill_data_size = nullptr;
RuntimeProfile::Counter* _spill_block_count = nullptr;
+ RuntimeProfile::Counter* _spill_wait_in_queue_timer = nullptr;
RuntimeProfile::Counter* _spill_write_wait_io_timer = nullptr;
RuntimeProfile::Counter* _spill_read_wait_io_timer = nullptr;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/13: [fix](plsql) Fix plsql variable initialization (#33186)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b696909775edd63302eb9cfc72521833ec1c6b9c
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Wed Apr 3 16:35:17 2024 +0800
[fix](plsql) Fix plsql variable initialization (#33186)
---
.../antlr4/org/apache/doris/nereids/PLLexer.g4 | 4 -
.../antlr4/org/apache/doris/nereids/PLParser.g4 | 2 +-
.../src/main/java/org/apache/doris/plsql/Exec.java | 6 --
.../doris/plsql/executor/DorisRowResult.java | 2 +-
.../data/plsql_p0/test_plsql_variable.out | 13 +++
.../suites/plsql_p0/test_plsql_loop_cursor.groovy | 7 ++
.../suites/plsql_p0/test_plsql_variable.groovy | 110 +++++++++++++++++++++
7 files changed, 132 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLLexer.g4
index 9a0c060aa73..8ee19ae851b 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLLexer.g4
@@ -208,7 +208,3 @@ SYSDATE: 'SYSDATE';
VARIANCE: 'VARIANCE';
DOT2: '..';
-
-LABEL_PL
- : ([a-zA-Z] | DIGIT | '_')* ':'
- ;
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4
index b052818c29e..f8dc6039145 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/PLParser.g4
@@ -522,7 +522,7 @@ for_range_stmt : // FOR (Integer range) statement
;
label_stmt :
- LABEL_PL
+ IDENTIFIER COLON
| LT LT IDENTIFIER GT GT
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
index 3caf8181ff0..87a108299d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
@@ -2085,12 +2085,6 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
public Integer visitLabel_stmt(Label_stmtContext ctx) {
if (ctx.IDENTIFIER() != null) {
exec.labels.push(ctx.IDENTIFIER().toString());
- } else {
- String label = ctx.LABEL_PL().getText();
- if (label.endsWith(":")) {
- label = label.substring(0, label.length() - 1);
- }
- exec.labels.push(label);
}
return 0;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java
index f087a7e17cc..e286485fd42 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/DorisRowResult.java
@@ -59,7 +59,7 @@ public class DorisRowResult implements RowResult {
@Override
public boolean next() {
- if (eof) {
+ if (eof || coord == null) {
return false;
}
try {
diff --git a/regression-test/data/plsql_p0/test_plsql_variable.out b/regression-test/data/plsql_p0/test_plsql_variable.out
new file mode 100644
index 00000000000..2fc878d6fcf
--- /dev/null
+++ b/regression-test/data/plsql_p0/test_plsql_variable.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+0
+
+-- !select --
+777 4
+
+-- !select --
+0
+
+-- !select --
+0
+
diff --git a/regression-test/suites/plsql_p0/test_plsql_loop_cursor.groovy b/regression-test/suites/plsql_p0/test_plsql_loop_cursor.groovy
index 72297888aaa..85cd7f1858a 100644
--- a/regression-test/suites/plsql_p0/test_plsql_loop_cursor.groovy
+++ b/regression-test/suites/plsql_p0/test_plsql_loop_cursor.groovy
@@ -15,6 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+// Add PL-SQL regression test notice:
+// 1. JDBC does not support the execution of stored procedures that return results. You can only Into the execution
+// results into a variable or write them into a table, because when multiple result sets are returned, JDBC needs
+// to use the prepareCall statement to execute, otherwise the Statemnt of the returned result executes Finalize.
+// Send EOF Packet will report an error;
+// 2. The format of the result returned by Doris Statement is xxxx\n, xxxx\n, 2 rows affected (0.03 sec).
+// PL-SQL uses Print to print variable values in an unformatted format, and JDBC cannot easily obtain them. Real results.
suite("test_plsql_loop_cursor") {
def tableName = "plsql_tbl"
sql "DROP TABLE IF EXISTS ${tableName}"
diff --git a/regression-test/suites/plsql_p0/test_plsql_variable.groovy b/regression-test/suites/plsql_p0/test_plsql_variable.groovy
new file mode 100644
index 00000000000..8738524f08c
--- /dev/null
+++ b/regression-test/suites/plsql_p0/test_plsql_variable.groovy
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_plsql_variable") {
+ def tableName = "plsql_variable"
+ sql """
+ CREATE OR REPLACE PROCEDURE plsql_variable1()
+ BEGIN
+ DECLARE a STRING;
+ a:="hello world!";
+ PRINT a;
+ PRINT upper(a);
+
+ DECLARE b = 10;
+ PRINT b;
+ b = length(a);
+ PRINT b;
+
+ DECLARE c = a;
+ PRINT c;
+ c:=b;
+ PRINT c;
+ c = "hello kudo!"
+ PRINT c;
+
+ DECLARE d STRING;
+ PRINT d;
+ d = NOW();
+ PRINT d;
+ END;
+ """
+ qt_select """call plsql_variable1()"""
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ create table ${tableName} (id int, name varchar(20)) DUPLICATE key(`id`) distributed by hash (`id`) buckets 4
+ properties ("replication_num"="1");
+ """
+
+ sql """
+ CREATE OR REPLACE PROCEDURE procedure_insert(IN id int, IN name STRING)
+ BEGIN
+ INSERT INTO ${tableName} VALUES(id, name);
+ END;
+ """
+ sql """call procedure_insert(111, "plsql111")"""
+ sql """call procedure_insert(222, "plsql222")"""
+ sql """call procedure_insert(333, "plsql333")"""
+ sql """call procedure_insert(111, "plsql333")"""
+ qt_select "select sum(id), count(1) from ${tableName}"
+
+ sql """
+ CREATE OR REPLACE PROCEDURE plsql_variable2()
+ BEGIN
+ DECLARE a int = 2;
+ DECLARE b string = " plsql111 ";
+ print a;
+ print b;
+ print trim(b);
+
+ DECLARE c string;
+ select name into c from ${tableName} where 2=a and name=trim(b);
+ print c;
+
+ DECLARE d int;
+ select count(1) into d from ${tableName} where 2=a;
+ print d;
+ END;
+ """
+ qt_select """call plsql_variable2()"""
+
+ // TODO, currently, variable take priority over column, Oracle column priority.
+ sql """
+ CREATE OR REPLACE PROCEDURE plsql_variable3()
+ BEGIN
+ DECLARE a int = 999;
+ print a;
+
+ DECLARE b int;
+ select id into b from ${tableName} where 999=a limit 1;
+ print b;
+
+ DECLARE id int = 999;
+ print id;
+
+ DECLARE c string;
+ select id into c from ${tableName} where 999=id limit 1;
+ print c;
+
+ DECLARE d string;
+ select count(1) into d from ${tableName} where 999=id;
+ print d;
+ END;
+ """
+ qt_select """call plsql_variable3()"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/13: [Enhencement](Nereids) reduce child output rows if agg child is literal (#32188)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6798a24a27c16f9ec0c3f080eef4b65adf099ee1
Author: LiBinfeng <46...@users.noreply.github.com>
AuthorDate: Wed Apr 3 12:23:41 2024 +0800
[Enhencement](Nereids) reduce child output rows if agg child is literal (#32188)
with group by:
select max(1) from t1 group by c1; -> select 1 from (select c1 from t1 group by c1);
without group by:
select max(1) from t1; -> select max(1) from (select 1 from t1 limit 1) tmp;
---
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../rewrite/ReduceAggregateChildOutputRows.java | 89 ++
.../eliminate_aggregate_constant.out | 993 +++++++++++++++++++++
.../eliminate_aggregate_constant.groovy | 285 ++++++
5 files changed, 1370 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index a68c7510965..e8223524367 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -117,6 +117,7 @@ import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion;
import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion;
+import org.apache.doris.nereids.rules.rewrite.ReduceAggregateChildOutputRows;
import org.apache.doris.nereids.rules.rewrite.ReorderJoin;
import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren;
import org.apache.doris.nereids.rules.rewrite.SplitLimit;
@@ -203,6 +204,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new EliminateLimit(),
new EliminateFilter(),
new EliminateAggregate(),
+ new ReduceAggregateChildOutputRows(),
new EliminateJoinCondition(),
new EliminateAssertNumRows(),
new EliminateSemiJoin()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index aa38af04ec5..3a43e1ad672 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -239,6 +239,7 @@ public enum RuleType {
MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
+ REDUCE_AGGREGATE_CHILD_OUTPUT_ROWS(RuleTypeClass.REWRITE),
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReduceAggregateChildOutputRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReduceAggregateChildOutputRows.java
new file mode 100644
index 00000000000..94109879e27
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReduceAggregateChildOutputRows.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
+import org.apache.doris.nereids.trees.plans.LimitPhase;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Set;
+
+/** ReduceAggregateChildOutputRows
+ * with group by:
+ * select max(1) from t1 group by c1; -> select 1 from (select c1 from t1 group by c1);
+ * without group by:
+ * select max(1) from t1; -> select max(1) from (select 1 from t1 limit 1) tmp;
+ * */
+public class ReduceAggregateChildOutputRows extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalAggregate().then(agg -> {
+ Set<AggregateFunction> aggFunctions = agg.getAggregateFunctions();
+ // check whether we have aggregate(constant) in all aggregateFunctions
+ if (!(agg.child() instanceof LogicalLimit && ((LogicalLimit) agg.child()).getLimit() == 1)
+ || aggFunctions.isEmpty() || !aggFunctions.stream().allMatch(
+ f -> (f instanceof Min || f instanceof Max)
+ && (f.arity() == 1 && f.child(0).isConstant()))) {
+ return null;
+ }
+
+ ImmutableList.Builder<NamedExpression> newOutput = ImmutableList.builder();
+ for (int i = 0; i < agg.getOutputExpressions().size(); i++) {
+ NamedExpression expr = agg.getOutputExpressions().get(i);
+ if (expr instanceof Alias && expr.child(0) instanceof AggregateFunction) {
+ AggregateFunction f = (AggregateFunction) expr.child(0);
+ if (f instanceof Min || f instanceof Max) {
+ newOutput.add(new Alias(expr.getExprId(), f.child(0), expr.getName()));
+ } else {
+ throw new AnalysisException("Unexpected aggregate function: " + f);
+ }
+ } else {
+ newOutput.add(expr);
+ }
+ }
+
+ if (agg.getGroupByExpressions().isEmpty()) {
+ LogicalAggregate newAgg = new LogicalAggregate<>(agg.getGroupByExpressions(),
+ agg.getOutputExpressions(), new LogicalLimit(1, 0, LimitPhase.ORIGIN,
+ new LogicalProject<>(newOutput.build(), agg.child())));
+ return newAgg;
+ } else {
+ ImmutableList.Builder<NamedExpression> childOutput =
+ ImmutableList.builderWithExpectedSize(agg.getGroupByExpressions().size());
+ for (Expression expr : agg.getGroupByExpressions()) {
+ childOutput.add((NamedExpression) expr);
+ }
+ return new LogicalProject<>(newOutput.build(),
+ new LogicalAggregate<>(agg.getGroupByExpressions(), childOutput.build(), agg.child()));
+ }
+ }).toRule(RuleType.REDUCE_AGGREGATE_CHILD_OUTPUT_ROWS);
+ }
+
+}
diff --git a/regression-test/data/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.out b/regression-test/data/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.out
new file mode 100644
index 00000000000..c0d8526bc65
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.out
@@ -0,0 +1,993 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !basic_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_2 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_4 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_6 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_8 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_2_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_2 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_4 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_6 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_2_8 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_3_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_2 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_4 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_6 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_3_8 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_4_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_2 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_4 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_6 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_4_8 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_add_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_add_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !basic_add_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_add_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !basic_add_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_add_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !basic_add_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !basic_add_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !basic_add_2_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_add_2_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !basic_add_2_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_add_2_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !basic_add_2_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_add_2_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !basic_add_2_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !basic_add_2_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !basic_add_3_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_add_3_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !basic_add_3_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_add_3_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !basic_add_3_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_add_3_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !basic_add_3_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !basic_add_3_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !basic_add_4_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_add_4_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !basic_add_4_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_add_4_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !basic_add_4_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_add_4_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !basic_add_4_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !basic_add_4_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !add_sum_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !add_sum_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !add_sum_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !add_sum_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !add_sum_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !add_sum_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !add_sum_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t1]
+
+-- !add_sum_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t1]
+
+-- !add_sum_2_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !add_sum_2_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !add_sum_2_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !add_sum_2_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !add_sum_2_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !add_sum_2_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !add_sum_2_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t2]
+
+-- !add_sum_2_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t2]
+
+-- !add_sum_3_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !add_sum_3_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !add_sum_3_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !add_sum_3_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !add_sum_3_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !add_sum_3_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !add_sum_3_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t3]
+
+-- !add_sum_3_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t3]
+
+-- !add_sum_4_1 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !add_sum_4_2 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !add_sum_4_3 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !add_sum_4_4 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !add_sum_4_5 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !add_sum_4_6 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !add_sum_4_7 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[LOCAL]
+------PhysicalProject
+--------PhysicalOlapScan[t4]
+
+-- !add_sum_4_8 --
+PhysicalResultSink
+--PhysicalProject
+----hashAgg[GLOBAL]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------PhysicalOlapScan[t4]
+
+-- !basic_1 --
+1
+
+-- !basic_2 --
+1
+
+-- !basic_3 --
+1
+
+-- !basic_4 --
+1
+
+-- !basic_5 --
+1
+
+-- !basic_6 --
+1
+
+-- !basic_7 --
+1.0
+
+-- !basic_8 --
+1.0
+
+-- !basic_2_1 --
+1
+
+-- !basic_2_2 --
+1
+
+-- !basic_2_3 --
+1
+
+-- !basic_2_4 --
+1
+
+-- !basic_2_5 --
+1
+
+-- !basic_2_6 --
+1
+
+-- !basic_2_7 --
+1.0
+
+-- !basic_2_8 --
+1.0
+
+-- !basic_3_1 --
+
+-- !basic_3_2 --
+\N
+
+-- !basic_3_3 --
+
+-- !basic_3_4 --
+\N
+
+-- !basic_3_5 --
+
+-- !basic_3_6 --
+\N
+
+-- !basic_3_7 --
+
+-- !basic_3_8 --
+\N
+
+-- !basic_4_1 --
+1
+1
+
+-- !basic_4_2 --
+1
+
+-- !basic_4_3 --
+1
+1
+
+-- !basic_4_4 --
+1
+
+-- !basic_4_5 --
+1
+1
+
+-- !basic_4_6 --
+2
+
+-- !basic_4_7 --
+1.0
+1.0
+
+-- !basic_4_8 --
+1.0
+
+-- !basic_add_1 --
+2
+
+-- !basic_add_2 --
+2
+
+-- !basic_add_3 --
+2
+
+-- !basic_add_4 --
+2
+
+-- !basic_add_5 --
+2
+
+-- !basic_add_6 --
+2
+
+-- !basic_add_7 --
+2.0
+
+-- !basic_add_8 --
+2.0
+
+-- !basic_add_2_1 --
+2
+
+-- !basic_add_2_2 --
+2
+
+-- !basic_add_2_3 --
+2
+
+-- !basic_add_2_4 --
+2
+
+-- !basic_add_2_5 --
+2
+
+-- !basic_add_2_6 --
+2
+
+-- !basic_add_2_7 --
+2.0
+
+-- !basic_add_2_8 --
+2.0
+
+-- !basic_add_3_1 --
+
+-- !basic_add_3_2 --
+\N
+
+-- !basic_add_3_3 --
+
+-- !basic_add_3_4 --
+\N
+
+-- !basic_add_3_5 --
+
+-- !basic_add_3_6 --
+\N
+
+-- !basic_add_3_7 --
+
+-- !basic_add_3_8 --
+\N
+
+-- !basic_add_4_1 --
+2
+2
+
+-- !basic_add_4_2 --
+2
+
+-- !basic_add_4_3 --
+2
+2
+
+-- !basic_add_4_4 --
+2
+
+-- !basic_add_4_5 --
+2
+2
+
+-- !basic_add_4_6 --
+3
+
+-- !basic_add_4_7 --
+2.0
+2.0
+
+-- !basic_add_4_8 --
+2.0
+
+-- !add_sum_1 --
+3
+
+-- !add_sum_2 --
+3
+
+-- !add_sum_3 --
+3
+
+-- !add_sum_4 --
+3
+
+-- !add_sum_5 --
+3
+
+-- !add_sum_6 --
+3
+
+-- !add_sum_7 --
+3.0
+
+-- !add_sum_8 --
+3.0
+
+-- !add_sum_2_1 --
+3
+
+-- !add_sum_2_2 --
+3
+
+-- !add_sum_2_3 --
+3
+
+-- !add_sum_2_4 --
+3
+
+-- !add_sum_2_5 --
+3
+
+-- !add_sum_2_6 --
+3
+
+-- !add_sum_2_7 --
+3.0
+
+-- !add_sum_2_8 --
+3.0
+
+-- !add_sum_3_1 --
+
+-- !add_sum_3_2 --
+\N
+
+-- !add_sum_3_3 --
+
+-- !add_sum_3_4 --
+\N
+
+-- !add_sum_3_5 --
+
+-- !add_sum_3_6 --
+\N
+
+-- !add_sum_3_7 --
+
+-- !add_sum_3_8 --
+\N
+
+-- !add_sum_4_1 --
+3
+3
+
+-- !add_sum_4_2 --
+5
+
+-- !add_sum_4_3 --
+3
+3
+
+-- !add_sum_4_4 --
+5
+
+-- !add_sum_4_5 --
+3
+3
+
+-- !add_sum_4_6 --
+6
+
+-- !add_sum_4_7 --
+3.0
+3.0
+
+-- !add_sum_4_8 --
+5.0
+
diff --git a/regression-test/suites/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.groovy b/regression-test/suites/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.groovy
new file mode 100644
index 00000000000..e22b4087000
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/eliminate_aggregate_constant/eliminate_aggregate_constant.groovy
@@ -0,0 +1,285 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("eliminate_aggregate_constant") {
+ sql "SET enable_nereids_planner=true"
+ sql "set runtime_filter_mode=OFF"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "SET ignore_shape_nodes='PhysicalDistribute'"
+ sql 'DROP DATABASE IF EXISTS test_aggregate_constant'
+ sql 'CREATE DATABASE IF NOT EXISTS test_aggregate_constant'
+ sql 'use test_aggregate_constant'
+
+ // create tables
+ sql """drop table if exists t1;"""
+ sql """drop table if exists t2;"""
+ sql """drop table if exists t3;"""
+ sql """drop table if exists t4;"""
+
+ sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');"""
+ sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');"""
+
+ sql "insert into t1 values (101, 101)"
+ sql "insert into t2 values (null, null)"
+ sql "insert into t4 values (102, 102)"
+ sql "insert into t4 values (103, 103)"
+
+ /* ******** with one row ******** */
+ qt_basic_1 """explain shape plan select max(1) from t1 group by c1;"""
+ qt_basic_2 """explain shape plan select max(1) from t1;"""
+ qt_basic_3 """explain shape plan select min(1) from t1 group by c1;"""
+ qt_basic_4 """explain shape plan select min(1) from t1;"""
+ qt_basic_5 """explain shape plan select sum(1) from t1 group by c1;"""
+ qt_basic_6 """explain shape plan select sum(1) from t1;"""
+ qt_basic_7 """explain shape plan select avg(1) from t1 group by c1;"""
+ qt_basic_8 """explain shape plan select avg(1) from t1;"""
+
+ /* ******** with one row "null" ******** */
+ qt_basic_2_1 """explain shape plan select max(1) from t2 group by c2;"""
+ qt_basic_2_2 """explain shape plan select max(1) from t2;"""
+ qt_basic_2_3 """explain shape plan select min(1) from t2 group by c2;"""
+ qt_basic_2_4 """explain shape plan select min(1) from t2;"""
+ qt_basic_2_5 """explain shape plan select sum(1) from t2 group by c2;"""
+ qt_basic_2_6 """explain shape plan select sum(1) from t2;"""
+ qt_basic_2_7 """explain shape plan select avg(1) from t2 group by c2;"""
+ qt_basic_2_8 """explain shape plan select avg(1) from t2;"""
+
+ /* ******** with empty table ******** */
+ qt_basic_3_1 """explain shape plan select max(1) from t3 group by c3;"""
+ qt_basic_3_2 """explain shape plan select max(1) from t3;"""
+ qt_basic_3_3 """explain shape plan select min(1) from t3 group by c3;"""
+ qt_basic_3_4 """explain shape plan select min(1) from t3;"""
+ qt_basic_3_5 """explain shape plan select sum(1) from t3 group by c3;"""
+ qt_basic_3_6 """explain shape plan select sum(1) from t3;"""
+ qt_basic_3_7 """explain shape plan select avg(1) from t3 group by c3;"""
+ qt_basic_3_8 """explain shape plan select avg(1) from t3;"""
+
+ /* ******** with different group table ******** */
+ qt_basic_4_1 """explain shape plan select max(1) from t4 group by c4;"""
+ qt_basic_4_2 """explain shape plan select max(1) from t4;"""
+ qt_basic_4_3 """explain shape plan select min(1) from t4 group by c4;"""
+ qt_basic_4_4 """explain shape plan select min(1) from t4;"""
+ qt_basic_4_5 """explain shape plan select sum(1) from t4 group by c4;"""
+ qt_basic_4_6 """explain shape plan select sum(1) from t4;"""
+ qt_basic_4_7 """explain shape plan select avg(1) from t4 group by c4;"""
+ qt_basic_4_8 """explain shape plan select avg(1) from t4;"""
+
+ /* ******** with one row ******** */
+ qt_basic_add_1 """explain shape plan select max(1) + 1 from t1 group by c1;"""
+ qt_basic_add_2 """explain shape plan select max(1) + 1 from t1;"""
+ qt_basic_add_3 """explain shape plan select min(1) + 1 from t1 group by c1;"""
+ qt_basic_add_4 """explain shape plan select min(1) + 1 from t1;"""
+ qt_basic_add_5 """explain shape plan select sum(1) + 1 from t1 group by c1;"""
+ qt_basic_add_6 """explain shape plan select sum(1) + 1 from t1;"""
+ qt_basic_add_7 """explain shape plan select avg(1) + 1 from t1 group by c1;"""
+ qt_basic_add_8 """explain shape plan select avg(1) + 1 from t1;"""
+
+ /* ******** with one row "null" ******** */
+ qt_basic_add_2_1 """explain shape plan select max(1) + 1 from t2 group by c2;"""
+ qt_basic_add_2_2 """explain shape plan select max(1) + 1 from t2;"""
+ qt_basic_add_2_3 """explain shape plan select min(1) + 1 from t2 group by c2;"""
+ qt_basic_add_2_4 """explain shape plan select min(1) + 1 from t2;"""
+ qt_basic_add_2_5 """explain shape plan select sum(1) + 1 from t2 group by c2;"""
+ qt_basic_add_2_6 """explain shape plan select sum(1) + 1 from t2;"""
+ qt_basic_add_2_7 """explain shape plan select avg(1) + 1 from t2 group by c2;"""
+ qt_basic_add_2_8 """explain shape plan select avg(1) + 1 from t2;"""
+
+ /* ******** with empty table ******** */
+ qt_basic_add_3_1 """explain shape plan select max(1) + 1 from t3 group by c3;"""
+ qt_basic_add_3_2 """explain shape plan select max(1) + 1 from t3;"""
+ qt_basic_add_3_3 """explain shape plan select min(1) + 1 from t3 group by c3;"""
+ qt_basic_add_3_4 """explain shape plan select min(1) + 1 from t3;"""
+ qt_basic_add_3_5 """explain shape plan select sum(1) + 1 from t3 group by c3;"""
+ qt_basic_add_3_6 """explain shape plan select sum(1) + 1 from t3;"""
+ qt_basic_add_3_7 """explain shape plan select avg(1) + 1 from t3 group by c3;"""
+ qt_basic_add_3_8 """explain shape plan select avg(1) + 1 from t3;"""
+
+ /* ******** with different group table ******** */
+ qt_basic_add_4_1 """explain shape plan select max(1) + 1 from t4 group by c4;"""
+ qt_basic_add_4_2 """explain shape plan select max(1) + 1 from t4;"""
+ qt_basic_add_4_3 """explain shape plan select min(1) + 1 from t4 group by c4;"""
+ qt_basic_add_4_4 """explain shape plan select min(1) + 1 from t4;"""
+ qt_basic_add_4_5 """explain shape plan select sum(1) + 1 from t4 group by c4;"""
+ qt_basic_add_4_6 """explain shape plan select sum(1) + 1 from t4;"""
+ qt_basic_add_4_7 """explain shape plan select avg(1) + 1 from t4 group by c4;"""
+ qt_basic_add_4_8 """explain shape plan select avg(1) + 1 from t4;"""
+
+ /* ******** with one row ******** */
+ qt_add_sum_1 """explain shape plan select max(1) + sum(2) from t1 group by c1;"""
+ qt_add_sum_2 """explain shape plan select max(1) + sum(2) from t1;"""
+ qt_add_sum_3 """explain shape plan select min(1) + sum(2) from t1 group by c1;"""
+ qt_add_sum_4 """explain shape plan select min(1) + sum(2) from t1;"""
+ qt_add_sum_5 """explain shape plan select sum(1) + sum(2) from t1 group by c1;"""
+ qt_add_sum_6 """explain shape plan select sum(1) + sum(2) from t1;"""
+ qt_add_sum_7 """explain shape plan select avg(1) + sum(2) from t1 group by c1;"""
+ qt_add_sum_8 """explain shape plan select avg(1) + sum(2) from t1;"""
+
+ /* ******** with one row "null" ******** */
+ qt_add_sum_2_1 """explain shape plan select max(1) + sum(2) from t2 group by c2;"""
+ qt_add_sum_2_2 """explain shape plan select max(1) + sum(2) from t2;"""
+ qt_add_sum_2_3 """explain shape plan select min(1) + sum(2) from t2 group by c2;"""
+ qt_add_sum_2_4 """explain shape plan select min(1) + sum(2) from t2;"""
+ qt_add_sum_2_5 """explain shape plan select sum(1) + sum(2) from t2 group by c2;"""
+ qt_add_sum_2_6 """explain shape plan select sum(1) + sum(2) from t2;"""
+ qt_add_sum_2_7 """explain shape plan select avg(1) + sum(2) from t2 group by c2;"""
+ qt_add_sum_2_8 """explain shape plan select avg(1) + sum(2) from t2;"""
+
+ /* ******** with empty table ******** */
+ qt_add_sum_3_1 """explain shape plan select max(1) + sum(2) from t3 group by c3;"""
+ qt_add_sum_3_2 """explain shape plan select max(1) + sum(2) from t3;"""
+ qt_add_sum_3_3 """explain shape plan select min(1) + sum(2) from t3 group by c3;"""
+ qt_add_sum_3_4 """explain shape plan select min(1) + sum(2) from t3;"""
+ qt_add_sum_3_5 """explain shape plan select sum(1) + sum(2) from t3 group by c3;"""
+ qt_add_sum_3_6 """explain shape plan select sum(1) + sum(2) from t3;"""
+ qt_add_sum_3_7 """explain shape plan select avg(1) + sum(2) from t3 group by c3;"""
+ qt_add_sum_3_8 """explain shape plan select avg(1) + sum(2) from t3;"""
+
+ /* ******** with different group table ******** */
+ qt_add_sum_4_1 """explain shape plan select max(1) + sum(2) from t4 group by c4;"""
+ qt_add_sum_4_2 """explain shape plan select max(1) + sum(2) from t4;"""
+ qt_add_sum_4_3 """explain shape plan select min(1) + sum(2) from t4 group by c4;"""
+ qt_add_sum_4_4 """explain shape plan select min(1) + sum(2) from t4;"""
+ qt_add_sum_4_5 """explain shape plan select sum(1) + sum(2) from t4 group by c4;"""
+ qt_add_sum_4_6 """explain shape plan select sum(1) + sum(2) from t4;"""
+ qt_add_sum_4_7 """explain shape plan select avg(1) + sum(2) from t4 group by c4;"""
+ qt_add_sum_4_8 """explain shape plan select avg(1) + sum(2) from t4;"""
+
+
+ /* ******** Output ******** */
+
+ /* ******** with one row ******** */
+ order_qt_basic_1 """select max(1) from t1 group by c1;"""
+ order_qt_basic_2 """select max(1) from t1;"""
+ order_qt_basic_3 """select min(1) from t1 group by c1;"""
+ order_qt_basic_4 """select min(1) from t1;"""
+ order_qt_basic_5 """select sum(1) from t1 group by c1;"""
+ order_qt_basic_6 """select sum(1) from t1;"""
+ order_qt_basic_7 """select avg(1) from t1 group by c1;"""
+ order_qt_basic_8 """select avg(1) from t1;"""
+
+ /* ******** with one row "null" ******** */
+ order_qt_basic_2_1 """select max(1) from t2 group by c2;"""
+ order_qt_basic_2_2 """select max(1) from t2;"""
+ order_qt_basic_2_3 """select min(1) from t2 group by c2;"""
+ order_qt_basic_2_4 """select min(1) from t2;"""
+ order_qt_basic_2_5 """select sum(1) from t2 group by c2;"""
+ order_qt_basic_2_6 """select sum(1) from t2;"""
+ order_qt_basic_2_7 """select avg(1) from t2 group by c2;"""
+ order_qt_basic_2_8 """select avg(1) from t2;"""
+
+ /* ******** with empty table ******** */
+ order_qt_basic_3_1 """select max(1) from t3 group by c3;"""
+ order_qt_basic_3_2 """select max(1) from t3;"""
+ order_qt_basic_3_3 """select min(1) from t3 group by c3;"""
+ order_qt_basic_3_4 """select min(1) from t3;"""
+ order_qt_basic_3_5 """select sum(1) from t3 group by c3;"""
+ order_qt_basic_3_6 """select sum(1) from t3;"""
+ order_qt_basic_3_7 """select avg(1) from t3 group by c3;"""
+ order_qt_basic_3_8 """select avg(1) from t3;"""
+
+ /* ******** with different group table ******** */
+ order_qt_basic_4_1 """select max(1) from t4 group by c4;"""
+ order_qt_basic_4_2 """select max(1) from t4;"""
+ order_qt_basic_4_3 """select min(1) from t4 group by c4;"""
+ order_qt_basic_4_4 """select min(1) from t4;"""
+ order_qt_basic_4_5 """select sum(1) from t4 group by c4;"""
+ order_qt_basic_4_6 """select sum(1) from t4;"""
+ order_qt_basic_4_7 """select avg(1) from t4 group by c4;"""
+ order_qt_basic_4_8 """select avg(1) from t4;"""
+
+ /* ******** with one row ******** */
+ order_qt_basic_add_1 """select max(1) + 1 from t1 group by c1;"""
+ order_qt_basic_add_2 """select max(1) + 1 from t1;"""
+ order_qt_basic_add_3 """select min(1) + 1 from t1 group by c1;"""
+ order_qt_basic_add_4 """select min(1) + 1 from t1;"""
+ order_qt_basic_add_5 """select sum(1) + 1 from t1 group by c1;"""
+ order_qt_basic_add_6 """select sum(1) + 1 from t1;"""
+ order_qt_basic_add_7 """select avg(1) + 1 from t1 group by c1;"""
+ order_qt_basic_add_8 """select avg(1) + 1 from t1;"""
+
+ /* ******** with one row "null" ******** */
+ order_qt_basic_add_2_1 """select max(1) + 1 from t2 group by c2;"""
+ order_qt_basic_add_2_2 """select max(1) + 1 from t2;"""
+ order_qt_basic_add_2_3 """select min(1) + 1 from t2 group by c2;"""
+ order_qt_basic_add_2_4 """select min(1) + 1 from t2;"""
+ order_qt_basic_add_2_5 """select sum(1) + 1 from t2 group by c2;"""
+ order_qt_basic_add_2_6 """select sum(1) + 1 from t2;"""
+ order_qt_basic_add_2_7 """select avg(1) + 1 from t2 group by c2;"""
+ order_qt_basic_add_2_8 """select avg(1) + 1 from t2;"""
+
+ /* ******** with empty table ******** */
+ order_qt_basic_add_3_1 """select max(1) + 1 from t3 group by c3;"""
+ order_qt_basic_add_3_2 """select max(1) + 1 from t3;"""
+ order_qt_basic_add_3_3 """select min(1) + 1 from t3 group by c3;"""
+ order_qt_basic_add_3_4 """select min(1) + 1 from t3;"""
+ order_qt_basic_add_3_5 """select sum(1) + 1 from t3 group by c3;"""
+ order_qt_basic_add_3_6 """select sum(1) + 1 from t3;"""
+ order_qt_basic_add_3_7 """select avg(1) + 1 from t3 group by c3;"""
+ order_qt_basic_add_3_8 """select avg(1) + 1 from t3;"""
+
+ /* ******** with different group table ******** */
+ order_qt_basic_add_4_1 """select max(1) + 1 from t4 group by c4;"""
+ order_qt_basic_add_4_2 """select max(1) + 1 from t4;"""
+ order_qt_basic_add_4_3 """select min(1) + 1 from t4 group by c4;"""
+ order_qt_basic_add_4_4 """select min(1) + 1 from t4;"""
+ order_qt_basic_add_4_5 """select sum(1) + 1 from t4 group by c4;"""
+ order_qt_basic_add_4_6 """select sum(1) + 1 from t4;"""
+ order_qt_basic_add_4_7 """select avg(1) + 1 from t4 group by c4;"""
+ order_qt_basic_add_4_8 """select avg(1) + 1 from t4;"""
+
+ /* ******** with one row ******** */
+ order_qt_add_sum_1 """select max(1) + sum(2) from t1 group by c1;"""
+ order_qt_add_sum_2 """select max(1) + sum(2) from t1;"""
+ order_qt_add_sum_3 """select min(1) + sum(2) from t1 group by c1;"""
+ order_qt_add_sum_4 """select min(1) + sum(2) from t1;"""
+ order_qt_add_sum_5 """select sum(1) + sum(2) from t1 group by c1;"""
+ order_qt_add_sum_6 """select sum(1) + sum(2) from t1;"""
+ order_qt_add_sum_7 """select avg(1) + sum(2) from t1 group by c1;"""
+ order_qt_add_sum_8 """select avg(1) + sum(2) from t1;"""
+
+ /* ******** with one row "null" ******** */
+ order_qt_add_sum_2_1 """select max(1) + sum(2) from t2 group by c2;"""
+ order_qt_add_sum_2_2 """select max(1) + sum(2) from t2;"""
+ order_qt_add_sum_2_3 """select min(1) + sum(2) from t2 group by c2;"""
+ order_qt_add_sum_2_4 """select min(1) + sum(2) from t2;"""
+ order_qt_add_sum_2_5 """select sum(1) + sum(2) from t2 group by c2;"""
+ order_qt_add_sum_2_6 """select sum(1) + sum(2) from t2;"""
+ order_qt_add_sum_2_7 """select avg(1) + sum(2) from t2 group by c2;"""
+ order_qt_add_sum_2_8 """select avg(1) + sum(2) from t2;"""
+
+ /* ******** with empty table ******** */
+ order_qt_add_sum_3_1 """select max(1) + sum(2) from t3 group by c3;"""
+ order_qt_add_sum_3_2 """select max(1) + sum(2) from t3;"""
+ order_qt_add_sum_3_3 """select min(1) + sum(2) from t3 group by c3;"""
+ order_qt_add_sum_3_4 """select min(1) + sum(2) from t3;"""
+ order_qt_add_sum_3_5 """select sum(1) + sum(2) from t3 group by c3;"""
+ order_qt_add_sum_3_6 """select sum(1) + sum(2) from t3;"""
+ order_qt_add_sum_3_7 """select avg(1) + sum(2) from t3 group by c3;"""
+ order_qt_add_sum_3_8 """select avg(1) + sum(2) from t3;"""
+
+ /* ******** with different group table ******** */
+ order_qt_add_sum_4_1 """select max(1) + sum(2) from t4 group by c4;"""
+ order_qt_add_sum_4_2 """select max(1) + sum(2) from t4;"""
+ order_qt_add_sum_4_3 """select min(1) + sum(2) from t4 group by c4;"""
+ order_qt_add_sum_4_4 """select min(1) + sum(2) from t4;"""
+ order_qt_add_sum_4_5 """select sum(1) + sum(2) from t4 group by c4;"""
+ order_qt_add_sum_4_6 """select sum(1) + sum(2) from t4;"""
+ order_qt_add_sum_4_7 """select avg(1) + sum(2) from t4 group by c4;"""
+ order_qt_add_sum_4_8 """select avg(1) + sum(2) from t4;"""
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org