You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/17 11:57:18 UTC
[doris] branch branch-2.0 updated (022be4663f9 -> 1016ed2f2ec)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 022be4663f9 [fix](Nereids) merge file TVFProperties and Properties by mistake (#25534)
new 9c483f1ceba [enhancement](invert index) read columns by index reduce seek time (#24735)
new ecab2530b7a [feature](nereids)print query id with memo and physical tree (#25501)
new 02b581cb112 [chore](config) turnoff fuzzy for `enable_simdjson_reader` (#25521)
new 1016ed2f2ec [fix](Nereids) non-slot filter should not be push through aggregate (#25525)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.cpp | 1 -
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 1 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 41 +++---
be/src/vec/columns/column_fixed_length_object.h | 17 +++
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +-
.../rewrite/PushdownFilterThroughAggregation.java | 8 +-
.../test_nestedtypes_json_insert_into_with_s3.out | 139 +++++++++++++++++++++
.../load_p0/stream_load/invalid_nest_json2.json | 2 +-
.../filter_push_through_aggregate.out} | 2 +-
.../filter_push_through_aggregate.groovy} | 39 +++---
10 files changed, 207 insertions(+), 47 deletions(-)
create mode 100644 regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.out
copy regression-test/data/{correctness/test_constant_push_down_through_outer_join.out => nereids_rules_p0/filter_push_down/filter_push_through_aggregate.out} (73%)
copy regression-test/suites/{nereids_syntax_p0/select_const.groovy => nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy} (61%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/04: [enhancement](invert index) read columns by index reduce seek time (#24735)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9c483f1ceba861236624eeab8d7a13250e9b8466
Author: zzzxl <33...@users.noreply.github.com>
AuthorDate: Tue Oct 17 10:34:33 2023 +0800
[enhancement](invert index) read columns by index reduce seek time (#24735)
---
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 1 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 41 +++++++++++++---------
be/src/vec/columns/column_fixed_length_object.h | 17 +++++++++
3 files changed, 43 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index f075a5a2083..4f3ffff2961 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -284,6 +284,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, vectorized::MutableColumnPtr
Status BinaryDictPageDecoder::read_by_rowids(const rowid_t* rowids, ordinal_t page_first_ordinal,
size_t* n, vectorized::MutableColumnPtr& dst) {
if (_encoding_type == PLAIN_ENCODING) {
+ dst = dst->convert_to_predicate_column_if_dictionary();
return _data_page_decoder->read_by_rowids(rowids, page_first_ordinal, n, dst);
}
DCHECK(_parsed);
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 6bdc48b4723..548de6a15dd 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1588,28 +1588,19 @@ void SegmentIterator::_output_non_pred_columns(vectorized::Block* block) {
Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read,
bool set_block_rowid) {
SCOPED_RAW_TIMER(&_opts.stats->first_read_ns);
+
do {
- uint32_t range_from;
- uint32_t range_to;
+ uint32_t range_from = 0;
+ uint32_t range_to = 0;
bool has_next_range =
_range_iter->next_range(nrows_read_limit - nrows_read, &range_from, &range_to);
if (!has_next_range) {
break;
}
- if (_cur_rowid == 0 || _cur_rowid != range_from) {
- _cur_rowid = range_from;
- _opts.stats->block_first_read_seek_num += 1;
- if (_opts.runtime_state && _opts.runtime_state->enable_profile()) {
- SCOPED_RAW_TIMER(&_opts.stats->block_first_read_seek_ns);
- RETURN_IF_ERROR(_seek_columns(_first_read_column_ids, _cur_rowid));
- } else {
- RETURN_IF_ERROR(_seek_columns(_first_read_column_ids, _cur_rowid));
- }
- }
+
size_t rows_to_read = range_to - range_from;
- RETURN_IF_ERROR(
- _read_columns(_first_read_column_ids, _current_return_columns, rows_to_read));
- _cur_rowid += rows_to_read;
+ _cur_rowid = range_to;
+
if (set_block_rowid) {
// Here use std::iota is better performance than for-loop, maybe for-loop is not vectorized
auto start = _block_rowids.data() + nrows_read;
@@ -1621,8 +1612,26 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32
}
_split_row_ranges.emplace_back(std::pair {range_from, range_to});
- // if _opts.read_orderby_key_reverse is true, only read one range for fast reverse purpose
} while (nrows_read < nrows_read_limit && !_opts.read_orderby_key_reverse);
+
+ for (auto cid : _first_read_column_ids) {
+ auto& column = _current_return_columns[cid];
+ if (_prune_column(cid, column, true, nrows_read)) {
+ continue;
+ }
+ for (auto& range : _split_row_ranges) {
+ size_t nrows = range.second - range.first;
+
+ RETURN_IF_ERROR(_column_iterators[cid]->seek_to_ordinal(range.first));
+ size_t rows_read = nrows;
+ RETURN_IF_ERROR(_column_iterators[cid]->next_batch(&rows_read, column));
+ if (rows_read != nrows) {
+ return Status::Error<ErrorCode::INTERNAL_ERROR>("nrows({}) != rows_read({})", nrows,
+ rows_read);
+ }
+ }
+ }
+
return Status::OK();
}
diff --git a/be/src/vec/columns/column_fixed_length_object.h b/be/src/vec/columns/column_fixed_length_object.h
index 323fabd6256..42be6835c66 100644
--- a/be/src/vec/columns/column_fixed_length_object.h
+++ b/be/src/vec/columns/column_fixed_length_object.h
@@ -284,6 +284,23 @@ public:
memcpy(_data.data() + old_size, data + begin_offset, total_mem_size);
}
+ void insert_many_binary_data(char* data_array, uint32_t* len_array,
+ uint32_t* start_offset_array, size_t num) override {
+ if (UNLIKELY(num == 0)) {
+ return;
+ }
+
+ size_t old_count = _item_count;
+ resize(old_count + num);
+ auto dst = _data.data() + old_count * _item_size;
+ for (size_t i = 0; i < num; i++) {
+ auto src = data_array + start_offset_array[i];
+ uint32_t len = len_array[i];
+ dst += i * _item_size;
+ memcpy(dst, src, len);
+ }
+ }
+
protected:
size_t _item_size;
size_t _item_count;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/04: [chore](config) turnoff fuzzy for `enable_simdjson_reader` (#25521)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 02b581cb1128c0a6ac69871c84570feb6ee2fe6f
Author: lihangyu <15...@163.com>
AuthorDate: Tue Oct 17 18:42:11 2023 +0800
[chore](config) turnoff fuzzy for `enable_simdjson_reader` (#25521)
---
be/src/common/config.cpp | 1 -
.../test_nestedtypes_json_insert_into_with_s3.out | 139 +++++++++++++++++++++
.../load_p0/stream_load/invalid_nest_json2.json | 2 +-
3 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index d3f03eed84c..f82bc241c29 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1502,7 +1502,6 @@ void set_fuzzy_configs() {
// random value true or false
set_fuzzy_config("disable_storage_page_cache", ((rand() % 2) == 0) ? "true" : "false");
set_fuzzy_config("enable_system_metrics", ((rand() % 2) == 0) ? "true" : "false");
- set_fuzzy_config("enable_simdjson_reader", ((rand() % 2) == 0) ? "true" : "false");
// random value from 8 to 48
// s = set_fuzzy_config("doris_scanner_thread_pool_thread_num", std::to_string((rand() % 41) + 8));
// LOG(INFO) << s.to_string();
diff --git a/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.out b/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.out
new file mode 100644
index 00000000000..21fb51ac176
--- /dev/null
+++ b/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.out
@@ -0,0 +1,139 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_arr_json_without_quote_s3 --
+0 [false,true,false,true,true,true,false,false,false,false,false,true,true,true,false,false,true,true,true,false,false,false,false,true,false,true,true,false,true,false,true,false,true,false,true,true,false,false,true,true,false,false,true,true,false,true,false,true,true,false,false,true,true,false,false,true,true,true,true,false,false,false,false,false,false,false,true,true,true,true,false,true,false,true,true,true,true,false,false,false,true,true,true,false,true,false,false,false,false [...]
+
+-- !sql_arr_json_without_quote_doris --
+0 [0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1] [NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NU [...]
+
+-- !sql_arr_json_without_quote_s3 --
+0 [[false,true],[true,true],[false,false],[],[false],[false],[],[],[],[false,false],[],[],[true],[true],[false],[true],[true],[],[true,false],[],[],[false],[false],[],[true,false],[false,true],[],[false,false],[true,false],[false,true],[false,false],[],[],[true],[true],[true],[],[],[false,true],[false,false],[],[],[true,false],[false],[false],[true],[],[],[],[true],[false,false],[true,false],[false,false],[false],[false,true],[],[false],[false,true],[false,false],[],[false],[true],[],[], [...]
+
+-- !sql_arr_json_without_quote_doris --
+0 [[0, 1], [1, 1], [0, 0], [], [0], [0], [], [], [], [0, 0], [], [], [1], [1], [0], [1], [1], [], [1, 0], [], [], [0], [0], [], [1, 0], [0, 1], [], [0, 0], [1, 0], [0, 1], [0, 0], [], [], [1], [1], [1], [], [], [0, 1], [0, 0], [], [], [1, 0], [0], [0], [1], [], [], [], [1], [0, 0], [1, 0], [0, 0], [0], [0, 1], [], [0], [0, 1], [0, 0], [], [0], [1], [], [], [], [1, 0], [], [], [1], [1], [], [], [1, 0], [1, 0], [0, 0], [], [], [0, 0], [0, 1], [1], [1], [1], [1, 0], [1], [], [0, 0], [], [0, [...]
+
+-- !sql_arr_json_without_quote_s3 --
+0 {"false":true,"true":true} {"-64":-114,"5":87,"-128":-34,"-59":100,"-54":-124,"-91":2,"-7":14,"-96":-88,"25":6,"-108":91,"46":89,"93":6,"-3":-21,"78":118,"29":103,"-127":5,"106":75,"121":-63,"84":-107,"-104":-53,"61":-14,"-44":98,"6":-50,"60":-120,"117":86,"38":50,"70":45,"33":39,"65":-43,"-119":-115,"-4":-68,"-36":80,"9":36,"53":95,"-55":-70,"77":115,"-19":-63,"13":60,"-51":23,"41":109,"73":-82,"32":-47,"34":122,"-42":-16,"44":17,"118":-103,"-101":14,"39":31,"-52":-98,"-106":-85,"48": [...]
+
+-- !sql_arr_json_without_quote_doris --
+0 {0:1, 1:1} {-64:-114, 5:87, -128:-34, -59:100, -54:-124, -91:2, -7:14, -96:-88, 25:6, -108:91, 46:89, 93:6, -3:-21, 78:118, 29:103, -127:5, 106:75, 121:-63, 84:-107, -104:-53, 61:-14, -44:98, 6:-50, 60:-120, 117:86, 38:50, 70:45, 33:39, 65:-43, -119:-115, -4:-68, -36:80, 9:36, 53:95, -55:-70, 77:115, -19:-63, 13:60, -51:23, 41:109, 73:-82, 32:-47, 34:122, -42:-16, 44:17, 118:-103, -101:14, 39:31, -52:-98, -106:-85, 48:126, 63:92, -46:1, -1:-89, 16:-38, -14:97, -21:-69, 11:-6, 72:57, -3 [...]
+
+-- !sql_arr_json_without_quote_s3 --
+0 [{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true,"true":false},{"false":true [...]
+
+-- !sql_arr_json_without_quote_doris --
+0 [{0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0}, {0:1, 1:0 [...]
+
+-- !sql_arr_json_without_quote_s3 --
+0 {"true":[true,false,false],"false":[false,true,false]} {"101":[-38,-115,35],"-59":[-113,-128,110],"10":[-78,-127,16],"56":[108,-40,124],"-22":[-63,-88,-24],"37":[-7,81,-20],"14":[-114,95,-3],"110":[71,-72,-84],"125":[1,110,56],"-40":[-13,19,80],"20":[-41,-115,107],"46":[82,-104,-30],"57":[116,68,88],"-104":[-64,119,31],"-72":[-65,-44,27],"1":[-127,-73,81],"74":[81,-82,-32],"-99":[-70,37,55],"60":[89,112,124],"-58":[-69,50,-56],"102":[27,-17,-9],"-68":[-112,20,91],"-43":[75,21,76],"33": [...]
+
+-- !sql_arr_json_without_quote_doris --
+0 {1:[1, 0, 0], 0:[0, 1, 0]} {101:[-38, -115, 35], -59:[-113, -128, 110], 10:[-78, -127, 16], 56:[108, -40, 124], -22:[-63, -88, -24], 37:[-7, 81, -20], 14:[-114, 95, -3], 110:[71, -72, -84], 125:[1, 110, 56], -40:[-13, 19, 80], 20:[-41, -115, 107], 46:[82, -104, -30], 57:[116, 68, 88], -104:[-64, 119, 31], -72:[-65, -44, 27], 1:[-127, -73, 81], 74:[81, -82, -32], -99:[-70, 37, 55], 60:[89, 112, 124], -58:[-69, 50, -56], 102:[27, -17, -9], -68:[-112, 20, 91], -43:[75, 21, 76], 33:[103, 6 [...]
+
+-- !sql_arr --
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+
+-- !sql_arr --
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+
+-- !sql_arr --
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+100
+
+-- !sql_map_arr --
+88
+83
+89
+83
+89
+87
+83
+82
+84
+82
+92
+86
+79
+85
+81
+81
+88
+86
+83
+80
+82
+87
+80
+82
+83
+
diff --git a/regression-test/data/load_p0/stream_load/invalid_nest_json2.json b/regression-test/data/load_p0/stream_load/invalid_nest_json2.json
index 95fbd7b541a..8f4fcf5b374 100644
--- a/regression-test/data/load_p0/stream_load/invalid_nest_json2.json
+++ b/regression-test/data/load_p0/stream_load/invalid_nest_json2.json
@@ -7,7 +7,7 @@
{"no": 5, "item": {"id": 5, "city": "guangzhou","code":2345675}}
{"no": 5, "item": {"id": 5, "city": ["guangzhou"],"code":2345675}}
{"no": 5, "item": {"id": 5, "city": {"guangzhou":1},"code":2345675}}
-{"no": 5, "item": {"id": 5, "city": ["guangzhou","code":2345675]]}
+{"no": 5, "item": {"id": 5, "city": '"guangzhou","code":2345675]]}
{"no": 5, "item": {"id": 5, "city": ["guangzhou","code":2345675}}
{"no": 5, "item": {"id": 5, "city": {"guangzhou","code":2345675}}
{"no": 5, "item": {"id": 5, "city": {"guangzhou":1,"code":2345675}}}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/04: [feature](nereids)print query id with memo and physical tree (#25501)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ecab2530b7ad7ce8de24732c6b3cb88248681a60
Author: minghong <en...@gmail.com>
AuthorDate: Tue Oct 17 18:06:11 2023 +0800
[feature](nereids)print query id with memo and physical tree (#25501)
print query id with memo and physical tree when dump_nereids_memo switched on. This is used for regression test.
---
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 7aba5058680..817362d4638 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -240,7 +240,7 @@ public class NereidsPlanner extends Planner {
// if chooseNthPlan failed, we could get memo to debug
if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) {
String memo = cascadesContext.getMemo().toString();
- LOG.info(memo);
+ LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + memo);
}
int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan();
@@ -249,7 +249,7 @@ public class NereidsPlanner extends Planner {
physicalPlan = postProcess(physicalPlan);
if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) {
String tree = physicalPlan.treeString();
- LOG.info(tree);
+ LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + tree);
}
if (explainLevel == ExplainLevel.OPTIMIZED_PLAN
|| explainLevel == ExplainLevel.ALL_PLAN
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/04: [fix](Nereids) non-slot filter should not be push through aggregate (#25525)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1016ed2f2ec04ba69ad42690e8829ca88a97f319
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Tue Oct 17 18:02:26 2023 +0800
[fix](Nereids) non-slot filter should not be push through aggregate (#25525)
---
.../rewrite/PushdownFilterThroughAggregation.java | 8 ++--
.../filter_push_through_aggregate.out | 3 ++
.../filter_push_through_aggregate.groovy | 43 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java
index 217834c4bdc..45fa1c645b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.util.PlanUtils;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.HashSet;
@@ -63,17 +64,18 @@ public class PushdownFilterThroughAggregation extends OneRewriteRuleFactory {
Set<Expression> filterPredicates = Sets.newHashSet();
filter.getConjuncts().forEach(conjunct -> {
Set<Slot> conjunctSlots = conjunct.getInputSlots();
- if (canPushDownSlots.containsAll(conjunctSlots)) {
+ // NOTICE: filter not contain slot should not be pushed. e.g. 'a' = 'b'
+ if (!conjunctSlots.isEmpty() && canPushDownSlots.containsAll(conjunctSlots)) {
pushDownPredicates.add(conjunct);
} else {
filterPredicates.add(conjunct);
}
});
- if (pushDownPredicates.size() == 0) {
+ if (pushDownPredicates.isEmpty()) {
return null;
}
Plan bottomFilter = new LogicalFilter<>(pushDownPredicates, aggregate.child(0));
- aggregate = (LogicalAggregate<Plan>) aggregate.withChildren(bottomFilter);
+ aggregate = aggregate.withChildren(ImmutableList.of(bottomFilter));
return PlanUtils.filterOrSelf(filterPredicates, aggregate);
}).toRule(RuleType.PUSHDOWN_PREDICATE_THROUGH_AGGREGATION);
}
diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.out b/regression-test/data/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.out
new file mode 100644
index 00000000000..816c5d8a9ca
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !do_not_push_no_slot_filter --
+
diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy
new file mode 100644
index 00000000000..dfb65dd2466
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("filter_push_through_aggregate") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ sql """
+ DROP TABLE IF EXISTS filter_push_through_aggregate
+ """
+ sql """
+ CREATE TABLE IF NOT EXISTS filter_push_through_aggregate(
+ `id` int(11) NULL,
+ `msg` text NULL
+ ) ENGINE = OLAP
+ DISTRIBUTED BY HASH(id) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql "set disable_nereids_rules='REWRITE_FILTER_EXPRESSION'"
+
+ // 'a' = 'b' should not be push down aggregate.
+ qt_do_not_push_no_slot_filter """
+ select * from (select 'a' a, sum(1) from filter_push_through_aggregate) t where a = 'b';
+ """
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org